path: root/arch-mirror

#!/bin/bash

if [ "$(whoami)" != 'http' ]; then
  if [ $# -ne 0 ]; then
    >&2 printf 'Too many arguments: %s.\n' "$#"
    >&2 printf 'usage: %s\n' "$0"
    exit 1
  fi
  exec su http -s /bin/bash -c "$0"
fi

mirror_dir='/srv/http/arch'

rsync_options='-rtlHp4 --safe-links --timeout=600 --contimeout=60 --delay-updates --no-motd --partial --exclude .~tmp~'

total_timeout=1200

if stty &>/dev/null; then
  rsync_verbose='-v'
  wget_verbose='-v'
else
  rsync_verbose=''
  wget_verbose='-nv'
fi

retrieve_listing() {
  local url="$1"
  case "${url%%://*}" in
    'rsync')
      timeout ${total_timeout} rsync "${url}" \
      | grep '^d' \
      | awk '{print $5}'
    ;;
    'https'|'http')
      curl -fSsL --max-time ${total_timeout} "${url}" \
      | sed '
        s@^.*<a href="\([^"]\+\)/">\1/</a>.*$@\1@
        t
        d
      '
    ;;
    *)
      >&2 printf 'Unknown protocol: %s\n' "${url%%://*}"
    ;;
  esac
}

retrieve_file() {
  local url="$1"
  local file="$2"
  case "${url%%://*}" in
    'rsync')
      timeout ${total_timeout} rsync -q "${url}" "${file}"
    ;;
    'https'|'http')
      curl -fSsL --max-time ${total_timeout} "${url}" -o "${file}"
    ;;
    *)
      >&2 printf 'Unknown protocol: %s\n' "${url%%://*}"
    ;;
  esac
}

retrieve_content() {
  local url="$1"
  local destination="$2"
  local extra_options
  local timeout

  if [ -z "${destination##*/archlinuxarm/}" ]; then
    extra_options='--exclude os'
  else
    unset extra_options
  fi

  if [ -z "${destination##*/archlinux32/}" ]; then
    timeout='timeout 36000'
  else
    timeout="timeout ${total_timeout}"
  fi

  case "${url%%://*}" in
    'rsync')
      ${timeout} rsync ${rsync_options} ${rsync_verbose} ${extra_options} "${url}" "${destination}" \
      || return $?
    ;;
    'https'|'http')
      ${timeout} wget --mirror --prefer-family=Ipv4 -nH -np ${wget_verbose} -P "${destination}" --cut-dirs $(
        echo "${url#*//}" \
        | tr -d '\n' \
        | tr '/' '\n' \
        | sed 1d \
        | wc -l
      ) "${url}" \
      || return $?
    ;;
    *)
      >&2 printf 'Unknown protocol: %s\n' "${url%%://*}"
    ;;
  esac
}

resolve_complete_mirror() {
  local host="$1"
  local pre="${host%%//*}//"
  host="${host#${pre}}"
  local post="/${host#*/}"
  host="${host%${post}}"

  {
    dig +short "${host}" A \
    | sed '
      /^[0-9]\+\(\.[0-9]\+\)\{3\}$/ !d
    '
    dig +short "${host}" AAAA \
    | sed '
      s/^[0-9a-f:]\+$/[\0]/
      t
      d
    '
  } \
  | awk '{print "'"${pre}"'" $1 "'"${post}"'"}' \
  | while read -r url; do
    retrieve_listing "${url}" \
    | grep -v '^\(merged$\|\.\)' \
    | sed 's@^\(archlinuxfr\|quarry\)$@\0/os@' \
    | awk '{print "/" $1 "/#'"${url}"'" $1 "/"}'
  done
}

fetch_archlinux_mirrors() {
  curl -fSs --max-time ${total_timeout} 'https://www.archlinux.org/mirrors/status/json/' \
  | sed -n '
    s@^[^[]*"urls": \[@@
    T
    s@][^[]*$@@
    T
    p
  ' \
  | sed '
    s/}, {/},\n{/g
  ' \
  | grep -F '"protocol": "rsync"' \
  | grep -F '"isos": true' \
  | grep -vwF arch.eckner.net \
  | grep '"country_code": "\(DE\|CZ\|NL\|CH\|AT\|DK\)"' \
  | {
    date -u +'%FT%TZ CUTCUTCUT' -d@$(($(date +%s)-3600))
    sed '
      s@^.*, "last_sync": "\([^"]\+\)",.*$@\1 \0@
    '
  } \
  | sort -k1r,1 \
  | sed '
    / CUTCUTCUT/,$ d
    s@^\S\+ {"url": "\([^"]\+\)".*$@/archlinux/#\1@
    t
    d
  '
}

fetch_lastsync() {
  local tmp_file=$(mktemp)
  while read -r proximity url; do
    rm -f "${tmp_file}"
    if [ -n "${1}" ]; then
      if ! retrieve_file "${url}${1}" "${tmp_file}" 2>/dev/null; then
        rm -f "${tmp_file}"
      fi
    fi
    for file in 'lastsync' 'lastupdate'; do
      if [ -s "${tmp_file}" ]; then
        break
      fi
      for dir in '' 'os/'; do
        if [ -s "${tmp_file}" ]; then
          break
        fi
        if ! retrieve_file "${url}${dir}${file}" "${tmp_file}" 2>/dev/null; then
          rm -f "${tmp_file}"
        fi
      done
    done
    if [ -s "${tmp_file}" ]; then
      printf '%s %s %s\n' \
        "$(head -n1 "${tmp_file}")" \
        "${proximity}" \
        "${url}"
    else
      printf '%s %s %s\n' \
        "$(($(date +%s) - 120 + 60*proximity))" \
        "${proximity}" \
        "${url}"
    fi
  done
  rm -f "${tmp_file}"
}

remove_unindexed_packages() {
  find "$1" \
    -type d \
    -name any \
    -printf '%p\n' \
  | while read -r any_repo; do
    {
      find "${any_repo%/*}" \
        -name 'any' -prune , \
        -type l \
        -name '*.pkg*' \
        -printf '%f\n' \
      | sed '
        s@^@'"${any_repo}"'/@
        p
      '
      find "${any_repo}" \
        -mindepth 1 \
        -maxdepth 1 \
        -printf '%p\n'
    } \
    | sort \
    | uniq -u \
    | xargs -r rm
  done
  find "$1" \
    -type d \
    -name pool \
    -printf '%p\n' \
  | while read -r pool; do
    {
      find "${pool%/*}" \
        -name 'pool' -prune , \
        -type l \
        -name '*.pkg*' \
        -readlink -e {} + \
        grep '^'"${pool}"'/'
      find "${pool}" \
        -type f \
        -printf '%p\n'
    } \
    | sort \
    | uniq -u \
    | xargs -r rm
  done
  find "$1" \
    -name '.~tmp~' -prune , \
    -name '*.db' -printf '%p\n' \
  | while read -r db; do
    db_file="${db##*/}"
    {
      {
        find "${db%/*}/" \
          \( \
            -name "${db_file}*" -o \
            -name "${db_file%db}files*" \
          \) \
          -printf '%p\n'
        bsdtar -Oxf "${db}" \
        | sed '
          /^%FILENAME%$/ {
            N
            s@^%FILENAME%\n@'"${db%/*}"'/@
            t
          }
          D
        ' \
        | sed '
          p
          s/$/.sig/
        '
      } \
      | sed 'p'
      find "${db%/*}/" \
        -mindepth 1 \
        -maxdepth 1 \
        -printf '%p\n'
    } \
    | sort \
    | uniq -u \
    | xargs -r rm
  done
  find "$1" \
    -type d \
    -name '.~tmp~' \
    -exec rm -rf --one-file-system {} +
}

if [ $# -eq 2 ] && [ "$1" = 'mirror' ]; then
  to_sync="$2"

  for file in 'lastsync' 'lastupdate'; do
    for dir in '' 'os/'; do
      if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
        break
      fi
    done
    if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
      break
    fi
  done

  >&2 printf 'synchronizing %s ...\n' "${to_sync}"

  {
    if [ -f "${mirror_dir}${to_sync}${dir}${file}" ]; then
      printf '%s 0 ME\n' "$(head -n1 "${mirror_dir}${to_sync}${dir}${file}")"
    else
      unset dir
      unset file
    fi
    { 
      sed 's/^/1 /' "${mirror_dir}/.tmp/close-mirrors"
      sed 's/^/2 /' "${mirror_dir}/.tmp/far-mirrors"
    } \
    | sed '
      s@^\([12] \)'"${to_sync}"'#@\1@
      t
      d
    ' \
    | fetch_lastsync "${dir}${file}"
  } \
  | sort -k1nr,1 -k2n,2 \
  | sed '/ ME$/,$ d' \
  | sort -k2n,2 -k1nr,1 \
  | {
    last_successful_level=0
    while read -r _ level url; do
      if [ "${level}" -le "${last_successful_level}" ]; then
        continue
      fi
      if retrieve_content "${url}" "${mirror_dir}${to_sync}"; then
        last_successful_level="${level}"
      fi
    done
    if [ "${last_successful_level}" -eq 2 ]; then
      >&2 printf 'clean up %s ...\n' "${to_sync}"
      remove_unindexed_packages "${mirror_dir}${to_sync}"
    fi
  }
  >&2 printf '%s done.\n' "${to_sync}"

  exit

fi

if [ $# -ne 0 ]; then
  >&2 printf 'usage:\n'
  >&2 printf '  %s\n' "$0"
  >&2 printf '( %s mirror /sub-destination/#proto://host/url/ )\n' "$0"
  exit 1
fi

mkdir -p "${mirror_dir}"'/.tmp'

exec 9> "${mirror_dir}/.arch-mirror.lock"
flock -n 9 || exit 0

{
  resolve_complete_mirror 'rsync://jeti100.ioq.uni-jena.de/archlinux-all/'
  fetch_archlinux_mirrors \
  | shuf -n10
  printf '%s\n' \
    '/archlinux/#rsync://ftp.gwdg.de/pub/linux/archlinux/' \
    '/archlinuxarm/#rsync://ftp.halifax.rwth-aachen.de/archlinux-arm/' \
    '/archlinuxfr/os/#https://repo.archlinux.fr/' \
    '/quarry/os/#https://pkgbuild.com/~anatolik/quarry/'
} \
> "${mirror_dir}/.tmp/far-mirrors"

resolve_complete_mirror 'rsync://arch.eckner.net/archlinux-all/' \
> "${mirror_dir}/.tmp/close-mirrors"

cat \
  "${mirror_dir}/.tmp/close-mirrors" \
  "${mirror_dir}/.tmp/far-mirrors" \
| sed 's/#.*$//' \
| sort -u \
| parallel -j0 --lb -L1 "${0}" mirror

>&2 printf 'refreshing symlinks ...\n'

find "${mirror_dir}/" \
  -name '.*' -prune , \
  \( -name '*.files.tar.?z' -o -name '*.db.tar.?z' \) -printf '%p\n' \
| sed '
  s@\.\(files\|db\)\.tar\.[xg]z$@@
' \
| sort -u \
| sed '
  s@\(/\([^/]\+\)/os/\([^/]\+\)\)/\2$@\1 \2/os/\3@
  t
  s@\(/\([^/]\+\)/\([^/]\+\)\)/\3$@\1 \3/os/\2@
  t
  d
' \
| while read -r link_target link_name; do
  link_target="$(
    echo "${link_name%/*}/" \
    | sed '
      s@[^/]\+/@../@g
    '
  )..${link_target#${mirror_dir}}"
  link_name="${mirror_dir}/merged/${link_name}"
  mkdir -p "${link_name%/*}"
  rm -f "${link_name}"
  ln -s "${link_target}" "${link_name}"
done

>&2 printf 'symlinks done.\n'

>&2 printf 'refreshing timestamps ...\n'

declare -A dates

while read -r var date; do
  if [ "${var}" != 'lastupdate' ] \
  && [ "${var}" != 'lastsync' ]; then
    continue
  fi
  if [ -z "${dates[${var}.min]}" ] \
  || [ "${dates[${var}.min]}" -gt "${date}" ]; then
    dates["${var}.min"]="${date}";
  fi
  if [ -z "${dates[${var}.max]}" ] \
  || [ "${dates[${var}.max]}" -lt "${date}" ]; then
    dates["${var}.max"]="${date}";
  fi
done < <(
  find "${mirror_dir}/" -maxdepth 3 \
    -name '.*' -prune , \
    -type f \( \
      -name 'lastsync' -o -name 'lastupdate' \
    \) \
    -printf '%f ' \
    -exec head -n1 {} \; \
    -printf '\n' \
  | grep -vxF ''
)

for key in "${!dates[@]}"; do
  printf '%s\n' "${dates["${key}"]}" \
  > "${mirror_dir}/${key}"
done

>&2 printf 'timestamps done.\n'