arch-mirror: download parallely by parallel

1 files changed, 54 insertions, 33 deletions

diff --git a/arch-mirror b/arch-mirror
index e12b3f4..9930ce2 100755
--- a/arch-mirror
+++ b/arch-mirror
@@ -1,6 +1,11 @@
 #!/bin/bash
 
 if [ "$(whoami)" != 'http' ]; then
+  if [ $# -ne 0 ]; then
+    >&2 printf 'Too many arguments: %s.\n' "$#"
+    >&2 printf 'usage: %s\n' "$0"
+    exit 1
+  fi
   exec su http -s /bin/bash -c "$0"
 fi
 
@@ -8,11 +13,6 @@ mirror_dir='/srv/http/arch'
 
 rsync_options='-rtlvH --safe-links --delete-after --progress -h --timeout=600 --contimeout=60 -p --delay-updates --no-motd --temp-dir='"${mirror_dir}"'/.tmp --exclude .tmp'
 
-mkdir -p "${mirror_dir}"'/.tmp'
-
-exec 9> "${mirror_dir}/.arch-mirror.lock"
-flock -n 9 || exit 0
-
 if ! stty &>/dev/null; then
   quiet="-q"
 fi
@@ -192,28 +192,8 @@ fetch_lastsync() {
   rm -f "${tmp_file}"
 }
 
-far_mirrors=(
-  $(
-    resolve_complete_mirror 'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
-    fetch_archlinux_mirrors \
-    | shuf -n10
-  )
-  '/archlinux/#rsync://ftp.gwdg.de/pub/linux/archlinux/'
-  '/archlinuxarm/#rsync://ftp.halifax.rwth-aachen.de/archlinux-arm/'
-  '/archlinuxfr/#https://repo.archlinux.fr/'
-  '/quarry/#https://pkgbuild.com/~anatolik/quarry/'
-)
-
-close_mirrors=(
-  $(
-    resolve_complete_mirror 'rsync://arch.eckner.net/archlinux/'
-  )
-)
-
-printf '%s\n' "${far_mirrors[@]}" "${close_mirrors[@]}" \
-| sed 's/#.*$//' \
-| sort -u \
-| while read -r to_sync; do
+if [ $# -eq 2 ] && [ "$1" = 'mirror' ]; then
+  to_sync="$2"
 
   if [ "${to_sync}" = '/archlinuxarm/' ]; then
     extra_options='--exclude os'
@@ -242,8 +222,8 @@ printf '%s\n' "${far_mirrors[@]}" "${close_mirrors[@]}" \
       unset file
     fi
     { 
-      printf '1 %s\n' "${close_mirrors[@]}"
-      printf '2 %s\n' "${far_mirrors[@]}"
+      sed 's/^/1 /' "${mirror_dir}/.tmp/close-mirrors"
+      sed 's/^/2 /' "${mirror_dir}/.tmp/far-mirrors"
     } \
     | sed '
       s@^\([12] \)'"${to_sync}"'#@\1@
@@ -267,11 +247,48 @@ printf '%s\n' "${far_mirrors[@]}" "${close_mirrors[@]}" \
     done
   }
 
-  >&2 printf 'done.\n'
+  >&2 printf '%s done.\n' "${to_sync}"
 
-done
+  exit
+
+fi
+
+if [ $# -ne 0 ]; then
+  >&2 printf 'usage:\n'
+  >&2 printf '  %s\n' "$0"
+  >&2 printf '( %s mirror /sub-destination/#proto://host/url/ )\n' "$0"
+  exit 1
+fi
+
+mkdir -p "${mirror_dir}"'/.tmp'
+
+exec 9> "${mirror_dir}/.arch-mirror.lock"
+flock -n 9 || exit 0
+
+{
+  resolve_complete_mirror 'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
+  fetch_archlinux_mirrors \
+  | shuf -n10
+  printf '%s\n' \
+    '/archlinux/#rsync://ftp.gwdg.de/pub/linux/archlinux/' \
+    '/archlinuxarm/#rsync://ftp.halifax.rwth-aachen.de/archlinux-arm/' \
+    '/archlinuxfr/#https://repo.archlinux.fr/' \
+    '/quarry/#https://pkgbuild.com/~anatolik/quarry/'
+} \
+> "${mirror_dir}/.tmp/far-mirrors"
+
+resolve_complete_mirror 'rsync://arch.eckner.net/' \
+> "${mirror_dir}/.tmp/close-mirrors"
+
+cat \
+  "${mirror_dir}/.tmp/close-mirrors" \
+  "${mirror_dir}/.tmp/far-mirrors" \
+| sed 's/#.*$//' \
+| sort -u \
+| parallel -j0 --lb -L1 "${0}" mirror
+
+>&2 printf 'refreshing symlinks ...\n'
 
-# refresh symlinks
 find "${mirror_dir}/" \
   -name '.*' -prune , \
   \( -name '*.files.tar.?z' -o -name '*.db.tar.?z' \) -printf '%p\n' \
@@ -299,7 +316,9 @@ find "${mirror_dir}/" \
   ln -s "${link_target}" "${link_name}"
 done
 
-# refresh lastsync & lastupdate
+>&2 printf 'symlinks done.\n'
+
+>&2 printf 'refreshing timestamps ...\n'
 
 declare -A dates
 
@@ -332,3 +351,5 @@ for key in "${!dates[@]}"; do
   printf '%s\n' "${dates["${key}"]}" \
   > "${mirror_dir}/${key}"
 done
+
+>&2 printf 'timestamps done.\n'