soweit erst mal lauffähig

6 files changed, 210 insertions, 153 deletions

diff --git a/arch-mirror b/arch-mirror
new file mode 100755
index 0000000..2204d25
--- /dev/null
+++ b/arch-mirror
@@ -0,0 +1,208 @@
+#!/bin/bash
+
+if [ "$(whoami)" != 'http' ]; then
+  exec su http -s /bin/bash -c "$0"
+fi
+
+mirror_dir='/srv/http/arch'
+
+rsync_options='-rtlvH --safe-links --delete-after --progress -h --timeout=600 --contimeout=60 -p --delay-updates --no-motd --temp-dir='"${mirror_dir}"'/.tmp --exclude .tmp'
+
+mkdir -p "${mirror_dir}"'/.tmp'
+
+exec 9> "${mirror_dir}/.arch-mirror.lock"
+flock -n 9 || exit 0
+
+if ! stty &>/dev/null; then
+  quiet="-q"
+fi
+
+resolve_complete_mirror() {
+  local host="$1"
+  local pre="${host%%//*}//"
+  host="${host#${pre}}"
+  local post="/${host#*/}"
+  host="${host%${post}}"
+
+  {
+    dig +short "${host}" A \
+    | sed '
+      /^[0-9]\+\(\.[0-9]\+\)\{3\}$/ !d
+    '
+    dig +short "${host}" AAAA \
+    | sed '
+      s/^[0-9a-f:]\+$/[\0]/
+      t
+      d
+    '
+  } \
+  | awk '{print "'"${pre}"'" $1 "'"${post}"'"}' \
+  | while read -r url; do
+    rsync ${quiet} "${url}" \
+    | grep '^d' \
+    | grep -v '\s\(merged$\|\.\)' \
+    | awk '{print "/" $5 "/#'"${url}"'" $5 "/"}'
+  done
+}
+
+fetch_lastsync() {
+  local tmp_file=$(mktemp)
+  while read -r proximity url; do
+    rm -f "${tmp_file}"
+    if [ -n "${1}" ]; then
+      if ! rsync -q "${url}${1}" "${tmp_file}" 2>/dev/null; then
+        rm -f "${tmp_file}"
+      fi
+    fi
+    for file in 'lastsync' 'lastupdate'; do
+      if [ -s "${tmp_file}" ]; then
+        break
+      fi
+      for dir in '' 'os/'; do
+        if [ -s "${tmp_file}" ]; then
+          break
+        fi
+        if ! rsync -q "${url}${dir}${file}" "${tmp_file}" 2>/dev/null; then
+          rm -f "${tmp_file}"
+        fi
+      done
+    done
+    if [ -s "${tmp_file}" ]; then
+      printf '%s %s %s\n' \
+        "$(head -n1 "${tmp_file}")" \
+        "${proximity}" \
+        "${url}"
+    else
+      printf '%s %s %s\n' \
+        "$(($(date +%s) - 120 + 60*proximity))" \
+        "${proximity}" \
+        "${url}"
+    fi
+  done
+  rm -f "${tmp_file}"
+}
+
+far_mirrors=(
+  $(
+    resolve_complete_mirror 'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
+
+    curl -Ss 'https://www.archlinux.org/mirrors/status/json/' \
+    | sed -n '
+      s@^[^[]*"urls": \[@@
+      T
+      s@][^[]*$@@
+      T
+      p
+    ' \
+    | sed '
+      s/}, {/},\n{/g
+    ' \
+    | grep -F '"protocol": "rsync"' \
+    | {
+      date -u +'%FT%TZ CUTCUTCUT' -d@$(($(date +%s)-3600))
+      sed '
+        s@^.*, "last_sync": "\([^"]\+\)",.*$@\1 \0@
+      '
+    } \
+    | sort -k1r,1 \
+    | sed '
+      / CUTCUTCUT/,$ d
+      s@^\S\+ {"url": "\([^"]\+\)".*$@2 /archlinux/#\1@
+      t
+      d
+    ' \
+    | shuf -n10
+  )
+  '/archlinux/#rsync://ftp.gwdg.de/pub/linux/archlinux/'
+  '/archlinuxarm/#rsync://ftp.halifax.rwth-aachen.de/archlinux-arm/'
+)
+
+close_mirrors=(
+  $(
+    resolve_complete_mirror 'rsync://arch.eckner.net/archlinux/'
+  )
+)
+
+printf '%s\n' "${far_mirrors[@]}" "${close_mirrors[@]}" \
+| sed 's/#.*$//' \
+| sort -u \
+| while read -r to_sync; do
+
+  for file in 'lastsync' 'lastupdate'; do
+    for dir in '' 'os/'; do
+      if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
+        break
+      fi
+    done
+    if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
+      break
+    fi
+  done
+
+  >&2 printf 'synchronizing %s ... ' "${to_sync}"
+
+  {
+    if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
+      printf '%s 0 ME\n' "$(head -n1 "${mirror_dir}${to_sync}${dir}${file}")"
+    else
+      unset dir
+      unset file
+    fi
+    { 
+      printf '1 %s\n' "${close_mirrors[@]}"
+      printf '2 %s\n' "${far_mirrors[@]}"
+    } \
+    | sed '
+      s@^\([12] \)'"${to_sync}"'#@\1@
+      t
+      d
+    ' \
+    | fetch_lastsync "${dir}${file}"
+  } \
+  | sort -k1nr,1 -k2n,2 \
+  | sed '/ ME$/,$ d' \
+  | sort -k2n,2 -k1nr,1 \
+  | {
+    last_successful_level=0
+    while read -r _ level url; do
+      if [ "${level}" -le "${last_successful_level}" ]; then
+        continue
+      fi
+      if rsync ${rsync_options} ${quiet} "${url}" "${mirror_dir}${to_sync}"; then
+        last_successful_level="${level}"
+      fi
+    done
+  }
+
+  >&2 printf 'done.\n'
+
+done
+
+# refresh symlinks
+find "${mirror_dir}" \
+  -name '.*' -prune , \
+  \( -name '*.files.tar.?z' -o -name '*.db.tar.?z' \) -printf '%p\n' \
+| sed '
+  s@\.\(files\|db\)\.tar\.[xg]z$@@
+' \
+| sort -u \
+| sed '
+  s@\(/\([^/]\+\)/os/\([^/]\+\)\)/\2$@\1 \2/os/\3@
+  t
+  s@\(/\([^/]\+\)/\([^/]\+\)\)/\3$@\1 \3/os/\2@
+  t
+  d
+' \
+| while read -r link_target link_name; do
+  link_target="$(
+    echo "${link_name%/*}/" \
+    | sed '
+      s@[^/]\+/@../@g
+    '
+  )..${link_target#${mirror_dir}}"
+  link_name="${mirror_dir}/merged/${link_name}"
+  mkdir -p "${link_name%/*}"
+  rm -f "${link_name}"
+  ln -s "${link_target}" "${link_name}"
+done
+
diff --git a/arch-spiegeln.service b/arch-mirror.service
index 52ba7e7..4f82d73 100644
--- a/arch-spiegeln.service
+++ b/arch-mirror.service
@@ -5,4 +5,4 @@ JoinsNamespaceOf=php-fpm.service
 [Service]
 PrivateTmp=True
 User=http
-ExecStart=/usr/bin/arch-spiegeln
+ExecStart=/usr/bin/arch-mirror
diff --git a/arch-spiegeln.timer b/arch-mirror.timer
index 600a27a..600a27a 100644
--- a/arch-spiegeln.timer
+++ b/arch-mirror.timer
diff --git a/arch-spiegeln b/arch-spiegeln
deleted file mode 100755
index 31fe055..0000000
--- a/arch-spiegeln
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/bin/bash
-
-#if [ "$(whoami)" != 'http' ]; then
-#  exec su http -s /bin/bash -c "$0"
-#fi
-
-mirror_dir='/srv/http/arch'
-mirror_dir='/tmp'
-
-upstream_mirrors=(
-  'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
-  '/archlinux/#rsync://ftp.gwdg.de/pub/linux/archlinux/'
-  '/archlinuxarm/#rsync://ftp.halifax.rwth-aachen.de/archlinux-arm/'
-)
-
-close_mirrors=(
-  'arch.eckner.net'
-)
-
-[ -r '/etc/arch-spiegeln.conf' ] \
-&& . '/etc/arch-spiegeln.conf'
-
-mkdir -p "${mirror_dir}/.locks"
-exec 9> "${mirror_dir}/.locks/arch-spiegeln.lock"
-flock -n 9 || exit 0
-
-if ! stty &>/dev/null; then
-  quiet="-q"
-fi
-
-#rsync_sources=(
-#  $(
-# $lastsync $url $vollständig $basis $lokalität (kleiner=lokaler)
-    {
-      printf '%s ME 1 / 0\n' $(
-        cat "${mirror_dir}/lastsync"
-      )
-      for close_mirror in "${close_mirrors[@]}"; do
-        {
-          dig +short "${close_mirror}" A \
-          | grep -x '[0-9]\+\(\.[0-9]\+\)\{3\}'
-          dig +short "${close_mirror}" AAAA \
-          | grep -x '[0-9a-f:]\+'
-        } \
-        | while read -r addr; do
-          printf '%s %s 1 / 1\n' \
-            $(
-              curl -Ss --resolve "${close_mirror}:443:${addr}" \
-              "https://${close_mirror}/lastsync"
-            ) \
-            $(
-              printf '%s\n' "${addr}" \
-              | sed '
-                s/^[0-9a-f:]\+$/[\0]/
-                s@^\S\+$@rsync://\0/archlinux/@
-              '
-            )
-        done
-      done
-      for upstream_mirror in "${upstream_mirrors[@]}"; do
-        if [ -n "${upstream_mirror%%*#*}" ]; then
-          basis='/'
-        else
-          basis="${upstream_mirror%%#*}"
-          upstream_mirror="${upstream_mirror#${basis}#}"
-        fi
-        tmp_file=$(mktemp)
-        if ! rsync "${upstream_mirror}lastsync" "${tmp_file}" 2>/dev/null; then
-          if [ -z "${upstream_mirror##*archlinux*arm*}" ]; then
-            date +%s >"${tmp_file}"
-          else
-            continue
-          fi
-        fi
-        printf '%s %s 1 %s 2\n' \
-          $(
-            cat "${tmp_file}"
-          ) \
-          "${upstream_mirror}" \
-          "${basis}"
-        rm "${tmp_file}"
-      done
-      curl -Ss 'https://www.archlinux.org/mirrors/status/json/' \
-      | sed -n '
-        s@^[^[]*"urls": \[@@
-        T
-        s@][^[]*$@@
-        T
-        p
-      ' \
-      | sed '
-        s/}, {/},\n{/g
-      ' \
-      | grep -F '"protocol": "rsync"' \
-      | {
-        date -u +'%FT%TZ CUTCUTCUT' -d@$(($(date +%s)-3600))
-        sed '
-          s@^.*, "last_sync": "\([^"]\+\)",.*$@\1 \0@
-        '
-      } \
-      | sort -k1r,1 \
-      | sed '
-        / CUTCUTCUT/,$ d
-        s@^\S\+ {"url": "\([^"]\+\)".*$@\1 0 /archlinux/ 2@
-        t
-        d
-      ' \
-      | shuf -n10 >/dev/null
-    } \
-    | sort -k1n,1 -k5n,5 -k3nr,3 \
-    | uniq --group=append -f2 \
-    | sed -n '
-      N
-      s/\n$//
-      t ok
-      D
-      b
-      :ok
-      p
-    ' \
-    | while read -r _ url vollst basis lokal; do
-      echo "$url $vollst $basis $lokal"
-    done
-#  )
-#)
-
-#printf '%s\n' "${rsync_sources[@]}"
-exit
-
-for rsync_source in "${rsync_sources[@]}"; do
-
-  rsync -rtlvH --safe-links --delete-after --progress -h ${quiet} --timeout=600 --contimeout=60 -p \
-    --delay-updates --no-motd --temp-dir=/srv/http/.tmp \
-    --exclude .tmp \
-    "${rsync_source}" \
-    '/srv/http/arch/'
-
-done
diff --git a/arch-spiegeln.conf.example b/arch-spiegeln.conf.example
deleted file mode 100644
index 8147c95..0000000
--- a/arch-spiegeln.conf.example
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-mirror_dir='/srv/http/arch'
-
-# complete upstream mirrors
-upstream_mirrors=(
-  'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
-)
-
-# geographically close mirrors identical to ours
-close_mirrors=(
-  'arch.eckner.net'
-)
diff --git a/sync.php b/sync.php
index 751dcf7..c84ac37 100644
--- a/sync.php
+++ b/sync.php
@@ -1,6 +1,6 @@
 <?php
 
-exec('screen -d -m /usr/bin/arch-spiegeln');
+exec('screen -d -m /usr/bin/arch-mirror');
 
 $f = fopen('/srv/http/arch/lastsync','r');
 $s = trim(fgets($f));