path: root/archPackagesUpdate

#!/bin/bash

# init variables
tmpDir="$(mktemp -d /tmp/archive.XXXXXX)"
trap 'rm -rf --one-file-system "${tmpDir}"' EXIT
pkgDirBase="/srv/arch-mirror/arch/arch"
pkgDir="${pkgDirBase}/archlinuxewe/os"
pkgSrcDir="$(dirname $(readlink -f -n "$0"))"
[ "$(hostname)" == "rechenknecht" ] && \
  [ "${pkgSrcDir}" == "${HOME}" ] && \
    aufRechenknecht=true || \
    aufRechenknecht=false
branch="HEAD"
force=false
upload=true
updateMe=true
abortOnMakepkgError=true
cleanCache=false
cleanUnsigned=false
checkDbSig='-v'
checkMakepkg='--check'
checkSanity=true
rmAddPkgs=false
log=''
logFile='-'
printOnly=false

# print help screen
usage()
{
  >&2 echo ''
  >&2 echo 'Skript zum Updaten der selbsterstellten Archlinux-Pakete'
  >&2 echo ''
  >&2 echo 'Optionen:'
  >&2 echo '    -b|--branch [commitish]:'
  >&2 echo '      branch to check out - defaults to "HEAD"'
  >&2 echo '    --cleanCache:'
  >&2 echo '      clean up pacman cache in build environments'
  >&2 echo '    -c|--cleanUnsigned:'
  >&2 echo '      remove unsigned Packages first'
  >&2 echo '    -f|--force:'
  >&2 echo '      rebuild packages even if they look up to date'
  >&2 echo '    --ignore-wrong-db-signature:'
  >&2 echo '      do not verify old database signature'
  >&2 echo '    -L|--log file:'
  >&2 echo '      generate logfile of makepkg-runs ("-" as file autogenerates name)'
  >&2 echo '    --no-abortOnMakepkgError:'
  >&2 echo '      do not abort when makepkg returns non-zero'
  >&2 echo '    --no-check:'
  >&2 echo '      pass --nocheck to makepkg'
  >&2 echo '    --no-sanity-check:'
  >&2 echo '      skip check for sanity of packages'
  >&2 echo '    --no-selfUpdate:'
  >&2 echo '      do not update this script from git'
  >&2 echo '    --no-upload:'
  >&2 echo '      do not upload built packages from rechenknecht to jeti100'
  >&2 echo '    --only $package:'
  >&2 echo '      only consider $package for build and update process'
  >&2 echo '    -p|--print-only:'
  >&2 echo '      only print what would be done'
  >&2 echo '    -r|--removeAdditionalPackages:'
  >&2 echo '      remove additional packages from build environments'
  >&2 echo ''
  exit 1
}

# clean up before exit
cleanUp () {
  if [ -n "${log}" ]
  then
    [ "a${logFile}" == 'a-' ] && logFile="/tmp/$(date '+%F-%H-%M-%S')"
    [[ "${logFile}" == *".tar.gz" ]] || logFile="${logFile}.tar.gz"
    cd "${tmpDir}"
    logFiles="$(
      for s in */*.log
      do
        [ -r "${s}" ] && echo "${s}"
      done | \
        tr "\n" " "
    )"
    if [ -n "${logFiles}" ]
    then
      tar -czf "${logFile}" ${logFiles}
      echo 'Logfiles saved to "'"${logFile}"'".'
    fi
  fi
}

echo 'parse arguments ...'
eval set -- "$(
  getopt -o b:cfL:pr \
    --long branch: \
    --long cleanCache \
    --long cleanUnsigned \
    --long force \
    --long ignore-wrong-db-signature \
    --long log: \
    --long no-abortOnMakepkgError \
    --long no-check \
    --long no-sanity-check \
    --long no-selfUpdate \
    --long no-upload \
    --long only: \
    --long print-only \
    --long removeAdditionalPackages \
    -n "$(basename "$0")" \
    -- "$@" \
    || echo "usage"
)"
args=("$@")

while true
do
  case $1 in
    -b|--branch)
      shift
      branch="$1"
      echo '"branch '"${branch}"'" erkannt'
    ;;
    --cleanCache)
      echo '"cleanCache" erkannt'
      cleanCache=true
    ;;
    -c|--cleanUnsigned)
      echo '"cleanUnsigned" erkannt'
      cleanUnsigned=true
    ;;
    -f|--force)
      echo '"force" erkannt'
      force=true
    ;;
    --ignore-wrong-db-signature)
      echo '"ignore-wrong-db-signature" erkannt'
      checkDbSig=''
    ;;
    -L|--log)
      shift
      log='-L'
      logFile="$1"
      echo '"log '"${logFile}"'" erkannt'
    ;;
    --no-abortOnMakepkgError)
      echo '"no-abortOnMakepkgError" erkannt'
      abortOnMakepkgError=false
    ;;
    --no-check)
      echo '"no-check" erkannt'
      checkMakepkg='--nocheck'
    ;;
    --no-sanity-check)
      echo '"no-sanity-check" erkannt'
      checkSanity=false
    ;;
    --no-selfUpdate)
      echo '"no-selfUpdate" erkannt'
      updateMe=false
    ;;
    --no-upload)
      echo '"no-upload" erkannt'
      upload=false
    ;;
    --only)
      shift
      echo '"only '"$1"'" erkannt'
      onlyPackages[${#onlyPackages[@]}]="$1"
    ;;
    -p|--print-only)
      echo '"print-only" erkannt'
      printOnly=true
    ;;
    -r|--removeAdditionalPackages)
      echo '"removeAdditionalPackages" erkannt'
      rmAddPkgs=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo "FEHLER: Verstehe Option \"$1\" doch nicht! Ich beende."
      exit 1
  esac
  shift
done

if [ ! $# -eq 0 ]
then
  >&2 echo 'FEHLER: Zu viele ('$#') Argumente:'
  for ((i=1; i<=$#; i++))
  do
    >&2 echo "'${!i}'"
  done
  >&2 echo 'Ich beende.'
  usage
fi

if ! ${checkSanity}
then
  if ${upload} || ${updateMe}
  then
    >&2 echo 'FEHLER: "--no-sanity-check" geht nur zusammen mit "--no-upload" und "--no-selfUpdate"! Ich beende.'
    exit 1
  fi
fi

echo '... done'

$aufRechenknecht && pkgSrcDir="${pkgSrcDir}/archPackages"
cd "${pkgSrcDir}"

# extract git ${branch}
git archive --format tar "${branch}" | \
  tar -x -C "${tmpDir}" -f -

cd "${tmpDir}"

# update $0
if ${updateMe} && ${aufRechenknecht} && [ -e "$(basename $0)" ] && ! diff -q "$(basename $0)" "$0" &> /dev/null
then
  cp "$(basename $0)" "$0"
  $0 "${args[@]}"
  err=$?
  exit ${err}
fi

# clean unsigned packages
if ${cleanUnsigned} && ${aufRechenknecht}
then
  ssh paule@127.0.0.1 '
    rmPaket() {
      [ "$(basename "$(pwd)")" == "any" ] || \
        repo-remove '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "${1%-*-*-*.pkg.tar.xz}"
      rm "$1"
      [ -e "$1.sig" ] && rm "$1.sig"
    }

    cd "'${pkgDir}'"
    for arch in *
    do
      [ -d "${arch}" ] || continue
      cd "${arch}"
      for paket in *.pkg.tar.xz
      do
        [ -e "${paket}.sig" ] && continue
        rmPaket ${paket}
      done
      cd ..
    done
  '
fi

unset pakete
unset pakete_teile
unset archs
unset verss

if ${aufRechenknecht}
then

# collect packages
  for paket in $( \
    echo */PKGBUILD | \
      sed "s|/PKGBUILD||g" \
    )
  do
    [ -d "${paket}" ] || continue
    [ ${#onlyPackages[@]} -eq 0 ] && consider=true || consider=false
    for ((i=0; i<${#onlyPackages[@]}; i++))
    do
      [[ "${paket}" == ${onlyPackages[${i}]} ]] && consider=true
    done
    ${consider} || continue

    unset epoch
    unset pkgname
    . ${paket}/PKGBUILD

    for singleArch in "${arch[@]}"; do

      [ "${singleArch}" == "x86_64" ] || \
        [ "${singleArch}" == "i686" ] || \
        [ "${singleArch}" == "any" ] || \
        continue

      allExist=true
      for pkgnam in "${pkgname[@]}"; do

        pkgFile="${pkgnam}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${singleArch}.pkg.tar.xz"

        [ "${singleArch}" == "any" ] && \
          for dir in ${pkgDir}/*
          do
            [ ! -d "${dir}" ] && continue
            [ ! -e "${dir}/${pkgFile}" ] && allExist=false
          done

        if [ ! -e "${pkgDir}/${singleArch}/${pkgFile}" ]; then
          allExist=false
        fi

      done

      if ${force} || \
        ! ${aufRechenknecht} || \
        ! ${allExist}
      then
        pakete[${#pakete[@]}]="${paket}"
        pakete_teile[${#pakete_teile[@]}]="${pkgname[@]}"
        archs[${#archs[@]}]="${singleArch}"
        verss[${#verss[@]}]="${epoch:+${epoch}:}${pkgver}-${pkgrel}"
      fi
    done
  done

# only print packages to be built?
  if ${printOnly}; then
    printf '%d scheduled builds:\n' "${#pakete[@]}"
    for ((i=0; i<${#pakete[@]}; i++)); do
      echo " ${pakete[${i}]} ${verss[${i}]} ${archs[${i}]}"
      printf '   %s\n' ${pakete_teile[${i}]}
    done
    exit 0
  fi

# check sanity of build-chroots
  err=false
  for chrootArch in 32 64
  do
    [ -d "/opt/arch${chrootArch}" ] || continue
    basePakete="$(
      (
        echo '
          pacman -Qg base base-devel | \
            cut -d " " -f 2 | \
            sort -u
          exit
        ' | \
          chrootUnpriv "${chrootArch}"
        printf '%s\n' \
          'update-all' 'ccache' 'git' 'mercurial' 'bzr' 'subversion'
      ) | \
        sort -u
    )"
    explizitinstalliertePakete="$(
      echo '
        pacman -Qtt | \
          cut -d " " -f 1
        exit
      ' | \
        chrootUnpriv "${chrootArch}" | \
        sed '
          /^msmtp$/d
          /^report-installed-packages$/d
          /^sendmailadvanced$/d
        '
    )"
    alleinstalliertenPakete="$(
      echo '
        pacman -Q | \
          cut -d " " -f 1
        exit
      ' | \
        chrootUnpriv "${chrootArch}"
    )"
    zuvielePakete="$(
      (
        (
          (
            echo "${explizitinstalliertePakete}"
            echo "${basePakete}"
          ) | \
            sort | \
            uniq -d
          echo "${explizitinstalliertePakete}"
        ) | \
          sort | \
          uniq -u
        echo '
          pacman -Qdt | \
            cut -d " " -f 1
          exit
        ' | \
          chrootUnpriv "${chrootArch}"
      ) | \
        sort -u
    )"
    zuwenigePakete="$(
      (
        (
          echo "${alleinstalliertenPakete}"
          echo "${basePakete}"
        ) | \
          sort | \
          uniq -d
        echo "${basePakete}"
      ) | \
        sort | \
        uniq -u
    )"

    if [ -n "${zuvielePakete}" ]
    then
      >&2 echo 'zu viele Pakete in "'/opt/arch${chrootArch}'" installiert:'
      >&2 echo "${zuvielePakete}"
      if ${rmAddPkgs}
      then
        echo '
          echo "'"${zuvielePakete}"'" | \
            xargs -n1 sudo pacman -Rs --noconfirm
          exit
        ' | \
          chrootUnpriv "${chrootArch}"
      else
        err=true
      fi
    fi
    if [ -n "${zuwenigePakete}" ]
    then
      >&2 echo 'zu wenige Pakete in "'/opt/arch${chrootArch}'" installiert:'
      >&2 echo "${zuwenigePakete}"
      err=true
    fi
  done

  if ${err}
  then
    cleanUp
    exit 1
  fi

  maxErr=0

# generate current makepkg.conf and makepkg32.conf
  cd "${tmpDir}"
  cp /etc/makepkg.conf .
  if ! patch -p0 -i makepkgSystem.patch
  then
    cleanUp
    exit 1
  fi
  sed 's|^\(BUILDENV=.*\)!sign|\1sign|' -i makepkg.conf
  cp makepkg{,32}.conf
  if ! patch -p0 -i makepkg32.patch
  then
    cleanUp
    exit 1
  fi

# update git sources
  find "$(
    grep '^SRCDEST=' makepkg.conf | \
      cut -d= -f2
  )" -mindepth 2 -maxdepth 2 -name 'HEAD' | \
    sed 's|/[^/]*$||' | \
    while read -r repo; do
      git -C "${repo}" fetch --all -p
    done

# update svn sources
  find "$(
    grep '^SRCDEST=' makepkg.conf | \
      cut -d= -f2
  )" -mindepth 2 -maxdepth 2 -name '.svn' | \
    sed 's|/[^/]*$||' | \
    while read -r repo; do
      svn update "${repo}"
    done

# build packages
  for ((i=0; i<${#pakete[@]}; i++))
  do
    paket="${pakete[${i}]}"
    paket_teile=(${pakete_teile[${i}]})
    arch="${archs[${i}]}"
    vers="${verss[${i}]}"

    cd "${tmpDir}/${paket}"
    rm -f ${paket}-*-${arch}.pkg.tar.xz*
    rm -rf --one-file-system src pkg

    # report installed packages at end of package() function - if the build process succeeded,
    # these packages should work
    echo '
      if [ "$(uname -m)" = "i686" ]; then
        eval "$(
          declare -f package | \
            sed '"'"'
              $ i sudo report-installed-packages archlinux-reports@eckner.net buildmaster@archlinux32.org
            '"'"'
        )"
      fi
    ' >> PKGBUILD

    if [ "${arch}" = 'i686' ]; then
      # 32-bit is built in chroot /opt/arch32
      (
        ${cleanCache} && \
          echo 'sudo pacman -Sc --noconfirm && \'
        echo '\
          sudo pacman -Sy && \
          cd "'"${tmpDir}/${paket}"'" && \
          _targetHost='\''--host=i686-pc-linux-gnu'\'' makepkg '${log} ${checkMakepkg}' --holdver --config "'"${tmpDir}"'/makepkg32.conf" -fsr --noconfirm --asdeps
          exit $?
        '
      ) | \
        chrootUnpriv 32
      err[${i}]=$?
    elif [ "${arch}" = 'x86_64' ] || [ "${arch}" = 'any' ]; then
      # 64-bit and any is built in chroot /opt/arch64
      (
        ${cleanCache} && \
          echo 'sudo pacman -Sc --noconfirm && \'
        echo '\
          sudo pacman -Sy && \
          cd "'"${tmpDir}/${paket}"'" && \
          makepkg '${log} ${checkMakepkg}' --holdver --config "'"${tmpDir}"'/makepkg.conf" -fsr --noconfirm --asdeps
          exit $?
        '
      ) | \
        chrootUnpriv 64
      err[${i}]=$?
    else
      printf 'unbekannte Architektur "%s"\n' "${arch}"
      err[${i}]=128
    fi
    for teil in "${paket_teile[@]}"; do
      if [ ${err[${i}]} -ne 0 ]; then
        break
      fi
      namcap "${tmpDir}/${paket}/${teil}-${vers}-${arch}.pkg.tar.xz" > \
        "${tmpDir}/namcap"
      err[${i}]=$?
      if [ -n "${log}" ]
      then
        cat "${tmpDir}/namcap" > \
          "${tmpDir}/${paket}/${teil}-namcap-${arch}.log"
      fi
      if [ ${err[${i}]} -ne 0 ]; then
        break
      fi
      if grep "$(
        {
          printf '%s\\|' \
            "E: ELF file (.*) found in an ['\"]any['\"] package"
          if ! grep -qF " ${teil} ist absichtlich nicht any ohne ELF Dateien" PKGBUILD; then
            printf '%s\\|' \
              "W: No ELF files and not an ['\"]any['\"] package"
          fi
        } | \
          sed 's@\\|$@@'
        )" "${tmpDir}/namcap"; then
        err[${i}]=64
      fi
    done
    rm -f "${tmpDir}/namcap"
    if ${abortOnMakepkgError} && [ ${err[${i}]} -ne 0 ]
    then
      cleanUp
      exit ${err[${i}]}
    fi
    [ ${err[${i}]} -gt ${maxErr} ] && maxErr=${err[${i}]}

    key="$(
      grep '^GPGKEY=' "${tmpDir}/makepkg.conf" | \
        sed 's|^GPGKEY=\s*"\(.*\)"\s*$|0x\1|'
    )"
    for teil in "${paket_teile[@]}"; do
      if ! gpg --verify ${teil}-${vers}-${arch}.pkg.tar.xz{.sig,} &> /dev/null; then
        rm -f "${teil}-${vers}-${arch}.pkg.tar.xz.sig"
        gpg --detach-sign -u "${key}" --no-armor ${teil}-${vers}-${arch}.pkg.tar.xz
      fi
    done

    if ${upload} && [ ${err[${i}]} -eq 0 ]
    then

# upload package and update db
      cd "${tmpDir}/${paket}"
      tar -cf - $(
        printf " %s-${vers}-${arch}.pkg.tar.xz"     "${paket_teile[@]}"
        printf " %s-${vers}-${arch}.pkg.tar.xz.sig" "${paket_teile[@]}"
      ) |
        ssh paule@127.0.0.1 '

          entferneAltePakete() {
            altePakete=$(
              ls -1 | \
                grep "^$1-[^-]\+-[^-]\+-$2\.pkg\.tar\.xz\$"
            )
            if [ -n "${altePakete}" ]
            then
              [ "$(basename "$(pwd)")" == "any" ] || \
                repo-remove '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "$1"
              for altesPaket in ${altePakete}
              do
                rm "${altesPaket}"
                if [ -e "${altesPaket}.sig" ] || [ -h "${altesPaket}.sig" ]
                then
                  rm "${altesPaket}.sig"
                fi
              done
            fi
          }

          addPaket() {
            [ "$(basename "$(pwd)")" == "any" ] || \
              repo-add '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "$1"
          }

          cd '${pkgDir}/${arch}/'
          for teil in '"${paket_teile[@]}"'; do
            entferneAltePakete "${teil}" "'${arch}'"
          done
          tar -xf -
          for teil in '"${paket_teile[@]}"'; do
            addPaket "${teil}-'"${vers}-${arch}"'.pkg.tar.xz"

            if [ "'${arch}'" == "any" ]
            then
              for lArch in $(ls ..)
              do
                [ ! -d "../${lArch}" ] && continue
                [ "${lArch}" == "any" ] && continue
                cd "../${lArch}"

                entferneAltePakete "${teil}" "'${arch}'"
                ln -s "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz" "${teil}-'${vers}-${arch}'.pkg.tar.xz"
                if [ -e "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz.sig" ]
                then
                  ln -s "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz.sig" "${teil}-'${vers}-${arch}'.pkg.tar.xz.sig"
                fi
                addPaket "${teil}-'${vers}-${arch}'.pkg.tar.xz"
              done
            fi
          done
        '
    fi
  done

# check db
  if ${upload}
  then
    ssh paule@127.0.0.1 '
      cd '${pkgDir}'
      err=0

      for arch in *
      do
        [ ! -d "${arch}" ] && continue
        [ "${arch}" == "any" ] && continue

        for paketFehler in $( \
          (
            tar -Oxzf '${pkgDir}'/${arch}/archlinuxewe.db.tar.gz | \
              grep -A1 "^%FILENAME%\$" | \
              grep -v "^%FILENAME%\$" | \
              grep -v -- "^--\$"
            ls '${pkgDir}'/${arch} | \
              grep "^.*\.pkg\.tar\.xz\$"
          ) | \
            sort | \
            uniq -u
        )
        do

          err=1
          if [ -e "'${pkgDir}'/${arch}/${paketFehler}" ]
          then
            >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur im Verzeichnisbaum vorhanden."
          else
            >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur in der Datenbank vorhanden."
          fi

        done

        for signaturWarnungen in $( \
          ls '${pkgDir}'/${arch} | \
            grep "^.*\.pkg\.tar\.xz\(\.sig\)\?\$" | \
            sed "s|\.sig\$||" | \
            sort | \
            uniq -u
        )
        do

          if [ -e "'${pkgDir}'/${arch}/${signaturWarnungen}" ]
          then
            >&2 echo "WARNUNG: Paket '${pkgDir}'/${arch}/${signaturWarnungen} hat keine Signatur."
          else
            err=1
            >&2 echo "FEHLER: Zur Signatur '${pkgDir}'/${arch}/${signaturWarnungen}.sig gibt es kein Paket."
          fi

        done

      done

      exit ${err}
    '
  fi

  cleanUp
  exit ${maxErr}

else

  if ${checkSanity}
  then

# check packages
    err=false

    namcap */PKGBUILD || err=true

    for paket in *
    do
      [ -r "${paket}/PKGBUILD" ] || continue

      if ! grep -q "^# Maintainer\s*:\s\+Erich Eckner\s\+<arch at eckner dot net>\$" ${paket}/PKGBUILD || \
         grep "^# Maintainer\s*:" ${paket}/PKGBUILD | grep -vq "^# Maintainer\s*:\s\+Erich Eckner\s\+<arch at eckner dot net>\$"
      then
        >&2 echo "${paket}/PKGBUILD hat falsche(n) Maintainer"
        err=true
      fi

      unset epoch
      . ${paket}/PKGBUILD

      if echo "${source}" | grep -q "://opensources\.eckner\.net/"
      then

# one of Erich's packages

        if grep -q "sha512sums" ${paket}/PKGBUILD
        then
          sumTyp="sha512sum"
          pkgSum="${sha512sums[@]}"
        elif grep -q "sha256sums" ${paket}/PKGBUILD
        then
          sumTyp="sha256sum"
          pkgSum="${sha256sums[@]}"
        fi
        remSum="$(curl "${source#*::}&post=${sumTyp}" 2> /dev/null)"
        if [ ! "${remSum}" == "${pkgSum}" ]
        then
          >&2 echo "${sumTyp}s von ${pkgname} stimmt nicht: ${remSum} vs. ${pkgSum}."
          err=true
        fi

      elif echo "${source}" | grep -q "://arch\.eckner\.net/"
      then

# a modified official package

        offVer="$( \
          {
            printf '%s-%s%s-%s-%s.pkg.tar.xz\n' \
              "${pkgname}" \
              "${epoch:+${epoch}:}" \
              "${pkgver}" \
              "${pkgrel}" \
              "${arch}"
            ssh paule@jeti100 'find '"${pkgDirBase}"' \( -type d \( -name armv6h -o -name armv7h -o -name aarch64 -o -name arm -o -name archlinux32 \) -prune -false \) -o -type f -name "'"${paket}"'-*.pkg.tar.xz" -exec basename "{}" \; '
          } | \
            sort -V | \
            tail -n1 \
          )"
        if [ ! "${offVer}" == "${pkgname}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${arch}.pkg.tar.xz" ]
        then
          >&2 echo "${pkgname} ist in den offiziellen Quellen aktueller: ${offVer}."
          err=true
        fi

      fi

    done
    if ${err}
    then
      exit 1
    fi

  fi

  cd "${pkgSrcDir}"
  git push
  err=$?
  if [ ${err} -ne 0 ]
  then
    exit ${err}
  fi

  ssh -p 22222 makepkg@141.35.51.219 "~/archPackagesUpdate" "${args[@]}"
  err=$?
  if [ ${err} -ne 0 ]
  then
    exit ${err}
  fi

fi