Parallel - mwicat/personal GitHub Wiki

Run script on all hosts

parallel -S host1,host2 --onall --basefile testscript ::: ./testscript > out

parallel --slf hostsfile --onall --basefile testscript ::: ./testscript > out

Distribute STDIN

cat giantfile.txt | parallel -j 8 --pipe mycommand

Distribute file

parallel -a giantfile.txt -j 8 --pipepart mycommand

parallel command

cat urls.lst | parallel -j100 wget {}

parallel -k echo ssh -n {} hostname ::: host{1..4}

• -k - keep same order

parallel scp to many hosts

parallel-scp -r -h ~/hosts/hosts.txt foo.txt /home/irb2/foo.txt

parallel ssh

       -P
       --print
              Display output as it arrives.  This option is of limited usefulness because output from different hosts are interleaved.
       -I
       --send-input
              Read input and send to each ssh process.  Since ssh allows a command script to be sent on standard input, the -I option may be used in lieu of the command argument.

parallel-ssh -P -h all_hosts grep test_host '/etc/apache2/sites-enabled/*'

parallel-ssh -P -h hosts.lst hostname

parallel-ssh -o results -H host1 -H host2 hostname

parallel --tagstring {} -k ssh -n {} hostname ::: host{1..4}

cat hosts | parallel --files --results outdir ssh -n {} 'cat /var/log/* | gzip'

parallel sniff

python parallel-ssh -h hosts -o results tcpdump -i any -s 0 -U -n -w - 'tcp port 80'

rsync from

parallel rsync -avz '{}:/var/log/apache2/other_vhosts_access.log*' {}/ << hosts

rsync to

cat hosts | parallel rsync -avz src {}:dest