path: root/bin/db-update

#!/bin/sh

# move binary packages from staging to testing (if possible [1]) and
# additionally all packages specified on the command line from testing
# to the respective stable repository

# 1] Condition for moving a package A from staging to testing is that:
#   a) nothing on the build-list depends on A and
#   b) no done package B which is not being moved depends on A

# TODO:

#  the meta data used to evaluate condition "b" might be for a different
#    version

#  correctly handle if multiple versions of a single package are marked
#    as "done" or "testing"

#  be (even?) more atomic

#  separate locks for staging, testing (and stable)

#  correctly handle if a package moved from community to official or
#    vice versa

# use "builds" instead of assuming package builds only it's name

# shellcheck disable=SC2039
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'db-update [options] [packages]:'
  >&2 echo ' move possible packages from staging to testing.'
  >&2 echo ' move packages on the command line from testing to stable.'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--block:       If necessary, wait for lock blocking.'
  >&2 echo '  -f|--from $file:  Read packages to move from testing to'
  >&2 echo '                    stable from $file (- is stdin).'
  >&2 echo '  -h|--help:        Show this help and exit.'
  >&2 echo '  -n|--no-action:   Only print what would be moved.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

# move_packages $package/$from_repository/$to_repository ...
# the existence of a directory $tmp_dir is assumed

move_packages() {

  if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
    >&2 echo 'move_packages: No tmp_dir provided.'
    exit 2
  fi

  local package
  local from_repo
  local to_repo
  local from_ending
  local to_ending
  local repo
  local part
  local dummynator
  local file

  rm -rf --one-file-system "${tmp_dir:?}/"*

  touch "${tmp_dir}/repos"
  touch "${tmp_dir}/packages"
  touch "${tmp_dir}/master-mirror-listing"
  mkdir "${tmp_dir}/transit"

  if ${no_action}; then
    dummynator='echo'
  else
    dummynator=''
  fi

  ls_master_mirror 'i686' | \
    while read -r repo; do
      ls_master_mirror "i686/${repo}" | \
        sed "s|^|i686/${repo}/|" >> \
        "${tmp_dir}/master-mirror-listing"
    done

  for package in "$@"; do
    if [ -z "${package}" ]; then
      continue
    fi

    to_repo="${package##*/}"
    package="${package%/*}"
    from_repo="${package##*/}"
    package="${package%/*}"

    if ${no_action}; then
      printf \
        'move "%s" from "%s" to "%s"\n' \
        "${package}" \
        "${from_repo}" \
        "${to_repo}"
    fi

    echo "${package}" >> \
      "${tmp_dir}/packages"

    if echo "${from_repo}" | \
      grep -q 'staging$' && \
      echo "${to_repo}" | \
        grep -q 'testing$'; then
      from_ending='done'
      to_ending='testing'
    elif echo "${from_repo}" | \
      grep -q 'testing$' && \
      ! echo "${to_repo}" | \
        grep -q 'testing$\|staging$'; then
      from_ending='testing'
      to_ending=''
    else
      >&2 printf 'move_packages: Cannot move package from "%s" to "%s".\n' "${from_repo}" "${to_repo}"
      exit 2
    fi

    echo "${from_repo}" > \
      "${tmp_dir}/${package}.from_repo"
    echo "${to_repo}" > \
      "${tmp_dir}/${package}.to_repo"
    echo "${from_ending}" > \
      "${tmp_dir}/${package}.from_ending"
    echo "${to_ending}" > \
      "${tmp_dir}/${package}.to_ending"

    if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
      >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}"
      exit 2
    fi

    cp \
      "${work_dir}/package-states/${package}.${from_ending}" \
      "${tmp_dir}/${package}.parts"

    sed \
      's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
      "${tmp_dir}/${package}.parts" > \
      "${tmp_dir}/${package}.parts_names"

    sed \
      'p;s|$|.sig|' \
      "${tmp_dir}/${package}.parts" > \
      "${tmp_dir}/${package}.parts_and_signatures"

    while read -r part; do
      if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/master-mirror-listing"; then
        >&2 printf \
          'move_packages: Cannot find file "%s", part of package "%s".\n' \
          "i686/${from_repo}/${part}" \
          "${package}"
        exit 2
      fi
    done < \
      "${tmp_dir}/${package}.parts"

    mkdir -p "${tmp_dir}/${from_repo}"
    mkdir -p "${tmp_dir}/${to_repo}"

    repos=$(
      # shellcheck disable=SC2046
      printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/repos") | \
        sort -u
    )
    echo "${repos}" > \
      "${tmp_dir}/repos"

  done

  if ${no_action}; then
    find "${tmp_dir}" -type f | \
      while read -r file; do
        if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] ||
          [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then
          echo "'${file}'"
        else
          echo "${file}:"
          sed 's|^|<<|;s|$|>>|' "${file}"
        fi
        echo
      done
  fi

  # receive the *.db.tar.gz's and *.files.tar.gz's

  while read -r repo; do

    ${master_mirror_rsync_command} \
      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
      "${tmp_dir}/${repo}/"

    # add and remove the packages locally

    if grep -qxF "${repo}" "${tmp_dir}/"*".from_repo"; then

      # shellcheck disable=SC2046
      repo-remove -q \
        "${tmp_dir}/${repo}/${repo}.db.tar.gz" \
        $(
          grep -lxF "${repo}" "${tmp_dir}/"*".from_repo" | \
            sed '
              s|\.from_repo$|.parts_names|
            ' | \
            xargs -rn1 cat
        )
    fi

    if grep -qxF "${repo}" "${tmp_dir}/"*".to_repo"; then
      grep -lxF "${repo}" "${tmp_dir}/"*".to_repo" | \
        sed '
          s|\.to_repo$||
        ' | \
        while read -r package; do
          while read -r part; do
            ${master_mirror_rsync_command} \
              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \
              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \
              "${tmp_dir}/transit/"
            repo-add -q \
              "${tmp_dir}/${repo}/${repo}.db.tar.gz" \
              "${tmp_dir}/transit/${part}"
            rm \
              "${tmp_dir}/transit/${part}" \
              "${tmp_dir}/transit/${part}.sig"
          done < \
            "${package}.parts"
        done
    fi

  done < "${tmp_dir}/repos"

  if ${no_action}; then
    find "${tmp_dir}" -type f
  fi

  # move the packages remotely via sftp

  (
    while read -r package; do

      if [ -z "${package}" ]; then
        continue
      fi

      while read -r part; do
        if [ -z "${part}" ]; then
          continue
        fi
        printf \
          'rename "%s" "%s"\n' \
          "i686/$(cat "${tmp_dir}/${package}.from_repo")/${part}" \
          "i686/$(cat "${tmp_dir}/${package}.to_repo")/${part}"
      done < \
        "${tmp_dir}/${package}.parts_and_signatures"

    done < \
      "${tmp_dir}/packages"
    echo 'quit'
  ) | \
    if ${no_action}; then
      sed 's|^|sftp: |'
    else
      ${master_mirror_sftp_command}
    fi

  # and push our local *.db.tar.gz via rsync

  while read -r repo; do

    # shellcheck disable=SC2086
    ${dummynator} ${master_mirror_rsync_command} \
      "${tmp_dir}/${repo}/${repo}.db."* \
      "${tmp_dir}/${repo}/${repo}.files."* \
      "${master_mirror_rsync_directory}/i686/${repo}/"

  done < \
    "${tmp_dir}/repos"

  while read -r package; do

    # then we can safely remove old versions

    while read -r part; do
      ${dummynator} remove_old_package_versions "i686/$(cat "${tmp_dir}/${package}.to_repo")" "${part}"
    done < \
      "${tmp_dir}/${package}.parts"

    # and update the state files

    from_ending=$(
      cat "${tmp_dir}/${package}.from_ending"
    )
    to_ending=$(
      cat "${tmp_dir}/${package}.to_ending"
    )

    if [ -z "${to_ending}" ]; then
      ${dummynator} rm \
        "${work_dir}/package-states/${package}.${from_ending}"
    else
      # remove old state files of $package with ending $to_ending
      find "${work_dir}/package-states" -maxdepth 1 | \
        grep "/$(str_to_regex "${package%.*.*.*}")\(\.[^.]\+\)\{3\}\.${to_ending}\$" | \
        xargs -rn1 ${dummynator} rm
      ${dummynator} mv \
        "${work_dir}/package-states/${package}.${from_ending}" \
        "${work_dir}/package-states/${package}.${to_ending}"
    fi

  done < \
    "${tmp_dir}/packages"

  if ! ${no_action}; then
    date '+%s' > \
      "${tmp_dir}/lastupdate"
    # shellcheck disable=SC2086
    ${dummynator} ${master_mirror_rsync_command} \
      "${tmp_dir}/lastupdate" \
      "${master_mirror_rsync_directory}/lastupdate"
  fi

  rm -rf --one-file-system "${tmp_dir:?}/"*

}

eval set -- "$(
  getopt -o bf:hn \
    --long block \
    --long from: \
    --long help \
    --long no-action \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
no_action=false

while true
do
  case "$1" in
    -b|--block)
      block_flag=''
    ;;
    -f|--from)
      shift
      if [ "x$1" = "x-" ]; then
        packages_to_stabilize=$(cat)
      else
        packages_to_stabilize=$(cat "$1")
      fi
    ;;
    -h|--help)
      usage 0
    ;;
    -n|--no-action)
      no_action=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

packages_to_stabilize=$(
  # shellcheck disable=SC2086
  printf '%s\n' \
    ${packages_to_stabilize} \
    "${@}"
)

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

tmp_dir=$(mktemp -d "${work_dir}/tmp.XXXXXX")
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT

packages_to_stabilize=$(
  (
    # shellcheck disable=SC2086
    printf '%s\n' ${packages_to_stabilize} | \
      sort -u | \
      sed '
        /\.pkg\.tar\.xz$/{
          w '"${tmp_dir}/packages-to-stabilize"'
          d
        }
      '
    find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -exec \
      grep -HF '' {} \; | \
      sed '
        s|^.*/||
        s|^\([^:]\+\)\.testing:|\1 |
      ' | \
      sort -k2,2 | \
      join -1 2 -2 1 -o 1.1 - "${tmp_dir}/packages-to-stabilize"
  ) | \
    sort -u
)

for package in ${packages_to_stabilize}; do
  # some sanity checks
  if [ ! -f "${work_dir}/package-states/${package}.testing" ]; then
    >&2 echo "Package '${package}' is not in testing!"
    exit 2
  fi
done

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${package_database_lock_file}"
if ! flock ${block_flag} 8; then
  >&2 echo 'come back (shortly) later - I cannot lock package database.'
  exit 1
fi

clean_up_lock_file() {
  rm -f "${package_database_lock_file}" "${build_list_lock_file}"
  rm -rf --one-file-system "${tmp_dir}"
}

trap clean_up_lock_file EXIT

# sanity check

for ending in 'done' 'testing'; do
  if [ "${ending}" = 'testing' ] && \
    [ -z "${packages_to_stabilize}" ]; then
    # if nothing is to be untested, we don't care about duplicate
    # testing packages (and maybe an unstaging fixes this anyway)
    continue
  fi
  if [ -n "$(
    find "${work_dir}/package-states" -name ".${ending}" -printf '%f\n' | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      sort | \
      uniq -d
    )" ]; then
    >&2 echo 'Removing duplicates not yet implemented:'
    find "${work_dir}/package-states" -name ".${ending}" -printf '%f\n' | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      sort | \
      uniq -d
    exit 42
  fi
done

# packages which can't be un-staged because they're still dependencies
# of any job on the build-list

(
  grep -vxF 'break_loops' "${work_dir}/build-list" | \
    while read -r pkg pkg_rev mod_rev repo; do
      generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
      cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.${repo}.depends"
    done | \
    sort -u

  find "${work_dir}/package-infos" -name '*.groups' \
    -exec grep -qx 'base\(-devel\)\?' {} \; \
    -printf '%f\n' | \
    sed '
      s|\(\.[^.]\+\)\{4\}$||
    ' | \
    sort -u > \
    "${tmp_dir}/base-packages"

  # no base / base-devel packages on the build list?
  if [ -z "$(
      (
        grep -vxF 'break_loops' "${work_dir}/build-list" | \
          awk '{print $1}' | \
          sort -u
        sort -u "${tmp_dir}/base-packages"
      ) | \
        sort | \
        uniq -d
    )" ]; then
    # unstage all base / base-devel packages from staging
    cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages"
  fi
) | \
  sort | \
  uniq -u > \
  "${tmp_dir}/keep-packages"

# packages which are done

find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
  sed '
    s|^\(.*\)\(\(\.[^.]\+\)\{3\}\)\.done$|\1 \1\2|
  ' | \
  sort > \
  "${tmp_dir}/done-packages"

# remove packages not yet done from keep-packages list

join -1 1 -2 1 -o 2.2 \
  "${tmp_dir}/keep-packages" \
  "${tmp_dir}/done-packages" | \
  sponge "${tmp_dir}/keep-packages"

# find all dependencies of the unstageable packages
mv \
  "${tmp_dir}/keep-packages" \
  "${tmp_dir}/new-keep-packages"
touch "${tmp_dir}/keep-packages"

while [ -s "${tmp_dir}/new-keep-packages" ]; do

  while read -r package; do
    generate_package_metadata "${package}"
  done < "${tmp_dir}/new-keep-packages"

  cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \
    sort -u | \
    sponge "${tmp_dir}/keep-packages"

  (
    while read -r package; do
      cat "${work_dir}/package-infos/${package}.depends"
    done < \
      "${tmp_dir}/new-keep-packages" | \
      sort -u
    cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages"
  ) | \
    sort | \
    uniq -u | \
    join -1 1 -2 1 -o 2.2 \
      - \
      "${tmp_dir}/done-packages" | \
    sponge "${tmp_dir}/new-keep-packages"

  # "new" is only what has not been there before
  cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \
    sort | \
    uniq -u | \
    sponge "${tmp_dir}/new-keep-packages"

done

# if build list is empty, remember all entries of 'deletion-list'
if grep -qvxF 'break_loops' "${work_dir}/build-list"; then
  delete_packages=''
else
  delete_packages=$(
    cat "${work_dir}/deletion-list"
  )
fi

if [ -z "${delete_packages}" ]; then

  # unlock build list

  rm -f "${build_list_lock_file}"
  flock -u 9

  clean_up_lock_file() {
    rm -rf --one-file-system "${tmp_dir}"
    rm -f "${package_database_lock_file}"
  }

fi

# calculate unstageable packages from keep_packages and done_packages

done_packages=$(
  (
    cut -d' ' -f2 < \
      "${tmp_dir}/done-packages"
    cat "${tmp_dir}/keep-packages"
  ) | \
    sort | \
    uniq -u
)

# move packages in packages_to_stabilize from *testing/ to the stable repos

# shellcheck disable=SC2046
move_packages $(

  for package in ${packages_to_stabilize}; do

    if [ -z "${package}" ]; then
      continue
    fi

    printf '%s/%s/%s\n' \
      "${package}" \
      "$(official_or_community "${package}" 'testing')" \
      "$(repository_of_package "${package}")"

  done

)

# move packages from *staging to *testing

# shellcheck disable=SC2046
move_packages $(

  for package in ${done_packages}; do

    if [ -z "${package}" ]; then
      continue
    fi

    printf '%s/%s/%s\n' \
      "${package}" \
      "$(official_or_community "${package}" 'staging')" \
      "$(official_or_community "${package}" 'testing')"

  done

)

# delete packages from deletion-list

if [ -n "${delete_packages}" ]; then
  echo "there is something to delete:"
  echo "${delete_packages}"
fi

clean_up_lock_file