bash で N行ずつバルク処理したい

ってあるよね？

とりあえず 1000件ほどの tsv を適当に作って

cat /dev/urandom | base64 -w0 | fold -w 10 | paste - - - | head -n 1000 > huge.tsv

$ head huge.tsv
Posc8lji4o  2cwngkuL+q  SwowWLktkr
0MLQqvEx24  y4ZfM375H4  KdIhH9itax
iGzqpBq95C  7BmgvHrEsW  z/gIhZgo14
14K8v6wTqI  mh30NtgYou  Tj6qx6pE2n
nNy4svr68R  jdsSB6FYiY  +IAAsN53BC
8Pwjh53HhV  73IIKzPz5v  2j1GKgu35l
yvxzKpsJia  8LrRjxfrFD  ywFhlezZPS
xiDZKQf0Da  1227YQU5ya  +fMcvOYe0j
6rQ78nGsJd  kAvubrqgTG  N4XRAVQ3TD
bEwk7AEFbZ  NoREai/dtu  K6nm7sAdir

10行ずつに区切って適当なヘッダを付けてみます

コツ

• export -f で function をexport して
• split の --filter にコールバックで指定する

a.sh

#!/bin/bash

function _bulk() {
    echo -e "c1\tc2\tc3"
    cat
}
export -f _bulk

cat huge.tsv | split -l 10 --filter _bulk

$ bash a.sh > out

$ wc -l out
1100 out

$ head -n 30 out
c1  c2  c3
Posc8lji4o  2cwngkuL+q  SwowWLktkr
0MLQqvEx24  y4ZfM375H4  KdIhH9itax
iGzqpBq95C  7BmgvHrEsW  z/gIhZgo14
14K8v6wTqI  mh30NtgYou  Tj6qx6pE2n
nNy4svr68R  jdsSB6FYiY  +IAAsN53BC
8Pwjh53HhV  73IIKzPz5v  2j1GKgu35l
yvxzKpsJia  8LrRjxfrFD  ywFhlezZPS
xiDZKQf0Da  1227YQU5ya  +fMcvOYe0j
6rQ78nGsJd  kAvubrqgTG  N4XRAVQ3TD
bEwk7AEFbZ  NoREai/dtu  K6nm7sAdir
c1  c2  c3
hWdqhFrK2B  hSbBavFqgR  idwa4ZvuZC
QrsXnSUXdN  k66wnHwi6k  g6DBRX14JX
izNslRLZVr  qmjCxAqvLG  lJ1nEwQNni
6UHA+nsKwW  gEE42phePd  gPnduaQKVn
QO3nKJi0Zs  jJcM/DfZJ5  PwwF9bocQS
NrvMzsIXWK  h4hnJyYt0p  20JVJBSJq5
22tRFn70W3  L0PG+uUpej  vPuPBws2m2
wyJhguYB1v  bH4HlBKl+4  UB6QaYntOe
o1YJ0c/O5t  fbiUKOiPBg  iKOOhDWRMK
aRu8DmegEW  Xf9PQWVhiS  KcRNN+vq//
c1  c2  c3
tDH9/+rB+z  PZCfl2p97S  wxbK/amAuK
rS4baCQ8SH  JS0pQujmvB  RnKEULeQL8
GX1DhRno99  HuY+vDUmyu  Qy2Pz/a/kz
QposxDvS9t  G27lJO64fZ  LLakF/D9bd
ExRSv1PSqv  kXDVf9WFyC  Y3TlhPPpIG
DLNn7NXVbM  EsZ54L1YW9  cvinfcaxwg
GNdpww4HU4  UkF5BjrXrc  SsDhVsy3bD

おわり。