wget_data.sh - Ecogenomics/GTDBNCBI GitHub Wiki

#!/bin/bash
# wget_data.sh — replacement for rsync_data.sh (NCBI retired rsync on 2026-06-01).
# Downloads each genome assembly directory over HTTPS, skipping files that are
# unchanged on the server (wget -N, via Last-Modified + size — verified against NCBI).
#
# Usage (unchanged from rsync_data.sh — drop-in with your existing GNU parallel call):
#   parallel --eta -j20 wget_data.sh '<(' echo {} ')' ... < bac120_genbank.lst
#
# $1 = a file containing one or more assembly-directory URLs, e.g.
#   https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/002/287/175/GCA_002287175.1_ASM228717v1/
#
# NOTE: rsync's --delete is NOT replicated here on purpose — genome removals are
# handled upstream by the `gtdb_migration_tk clean_ftp` step, before download.

set -u
input=$1
umask 022   # -> directories 755, files 644 (matches rsync --chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r)

while IFS= read -r url; do
    [ -z "$url" ] && continue
    echo ">>> $url"

    # Mirror the directory, preserving NCBI's path layout under ./genomes/
    #   host/genomes/all/GCA/002/.../ACC/  ->  ./genomes/all/GCA/002/.../ACC/
    #   -nH strips the hostname; --cut-dirs=1 strips the leading "genomes";
    #   -P genomes puts it back so the tree matches the old rsync layout exactly.
    wget -r -np -nH --cut-dirs=1 -P genomes \
         -N -e robots=off \
         --retry-connrefused --tries=0 --waitretry=45 --read-timeout=120 \
         -R "index.html*,README.txt" \
         --exclude-directories="*/all_assembly_versions,*/representative,*_assembly_structure" \
         "$url"

    status=$?
    if [ "$status" -eq 0 ]; then
        echo "wget completed normally: $url"
    else
        echo "wget FAILED (exit $status): $url" >&2
    fi
done < "$input"

# Enforce permissions rsync used to set via --chmod (umask covers new files;
# this fixes anything pre-existing).
find genomes -type d -exec chmod 755 {} + 2>/dev/null
find genomes -type f -exec chmod 644 {} + 2>/dev/null