access_BomAccessDocumentation_bom dp9 management_MDSS - ACCESS-NRI/accessdev-Trac-archive GitHub Wiki
MDSS migration
-
"man mdss" for command usage
-
Use copyq queue for migration of data to MDSS
-
If there are lots of small files please tar/gzip them up
-
An example script to migrate a year's worth of data, with one job submits itself to process the next month (run for a month first to work out the elapsed time required)
#!/bin/ksh
#PBS -P dp9
#PBS -q copyq
#PBS -e /home/548/ttl548/cpq.err
#PBS -o /home/548/ttl548/cpq.out
#PBS -l walltime=03:00:00
#PBS -l wd
#PBS -l storage=gdata/access+gdata/hh5+gdata/ig2
m=(0 31 28 31 30 31 30 31 31 30 31 30 31)
ymd=$(cat /home/548/ttl548/cron/cpq.date)
yy=$(( $ymd/10000 ))
typeset -RZ2 mm=$(( $ymd%10000/100 ))
end=$yy$mm${m[$mm]}
cd /g/data/ig2/usavm/1
while [ $ymd -le $end ]; do
mdss -P ig2 put -r $ymd usavm/1/$yy
ymd=$(/g/data/access/apps/cawcr-utils/1.0.0/bin/adddate $ymd 1)
# echo $ymd > /home/548/ttl548/cron/cpq.date
# qsub /home/548/ttl548/cron/cpq.sh
done
if [ $mm -lt 12 ]; then
typeset -RZ2 mm=$(( mm+1 ))
ymd=$yy${mm}01
echo $ymd > /home/548/ttl548/cron/cpq.date
qsub /home/548/ttl548/cron/cpq.sh
fi