diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh index 132909214..3723ac7aa 100644 --- a/src/bin/download_clone.sh +++ b/src/bin/download_clone.sh @@ -17,12 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with Overpass_API. If not, see . -EXEC_DIR="`pwd`/../" +set -euo pipefail + CLONE_DIR="$1" REMOTE_DIR= SOURCE= -DONE= META= +TEMP_FILE=/tmp/ovepass_files_list if [[ -z $1 ]]; then { @@ -52,9 +53,9 @@ process_param() fi }; -if [[ -n $1 ]]; then process_param $1; fi -if [[ -n $2 ]]; then process_param $2; fi -if [[ -n $3 ]]; then process_param $3; fi +if [[ -n ${1} ]]; then process_param "${1}"; fi +if [[ -n ${2} ]]; then process_param "${2}"; fi +if [[ -n ${3} ]]; then process_param "${3}"; fi FILES_BASE="\ nodes.bin nodes.map node_tags_local.bin node_tags_global.bin node_frequent_tags.bin node_keys.bin \ @@ -79,61 +80,68 @@ relation_changelog.bin relation_tags_local_attic.bin relation_tags_global_attic. # $2 - local destination fetch_file() { - wget -c -O "$2" "$1" + wget -c -O "${2}" "${1}" }; retry_fetch_file() { DEADLINE=$(($(date '+%s') + 86400)) - rm -f "$2" - fetch_file "$1" "$2" - until [[ -s "$2" ]]; do { - if [[ $(date '+%s') -ge $DEADLINE ]]; then - echo "File $1 unavailable. Aborting." + rm -f "${2}" + fetch_file "${1}" "${2}" + until [[ -s "${2}" ]]; do { + if [[ $(date '+%s' || true) -ge ${DEADLINE} ]]; then + echo "File ${1} unavailable. Aborting." exit 1 fi sleep 15 - fetch_file "$1" "$2" + fetch_file "${1}" "${2}" }; done }; -download_file() +parallel_download () { - echo - echo "Fetching $1" - retry_fetch_file "$REMOTE_DIR/$1" "$CLONE_DIR/$1" - echo "Fetching $1.idx" - retry_fetch_file "$REMOTE_DIR/$1.idx" "$CLONE_DIR/$1.idx" -} + aria2c -d "${CLONE_DIR}" -j 16 -R -x 16 -i "${TEMP_FILE}" +}; -mkdir -p "$CLONE_DIR" -fetch_file "$SOURCE/trigger_clone" "$CLONE_DIR/base-url" +mkdir -p "${CLONE_DIR}" +fetch_file "${SOURCE}/trigger_clone" "${CLONE_DIR}/base-url" -REMOTE_DIR=$(cat <"$CLONE_DIR/base-url") +REMOTE_DIR=$(cat < "${CLONE_DIR}/base-url") #echo "Triggered generation of a recent clone" #sleep 30 -retry_fetch_file "$REMOTE_DIR/replicate_id" "$CLONE_DIR/replicate_id" +retry_fetch_file "${REMOTE_DIR}/replicate_id" "${CLONE_DIR}/replicate_id" -for I in $FILES_BASE; do +rm -f "${TEMP_FILE}}" +touch "${TEMP_FILE}" +for I in ${FILES_BASE}; do { - download_file $I + echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}" + echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}" }; done +parallel_download -if [[ $META == "yes" || $META == "attic" ]]; then +rm -f "${TEMP_FILE}}" +if [[ ${META} == "yes" || ${META} == "attic" ]]; then { - for I in $FILES_META; do + for I in ${FILES_META}; do { - download_file $I + echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}" + echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}" }; done + parallel_download }; fi -if [[ $META == "attic" ]]; then +rm -f "${TEMP_FILE}}" +if [[ ${META} == "attic" ]]; then { - for I in $FILES_ATTIC; do + for I in ${FILES_ATTIC}; do { - download_file $I + echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}" + echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}" }; done + parallel_download }; fi +rm -f "${TEMP_FILE}}" echo " database ready."