path: root/archPackagesUpdate

#!/bin/bash

# init variables
tmpDir="$(mktemp -d /tmp/archive.XXXXXX)"
pkgDirBase="/srv/arch-mirror/arch/arch"
pkgDir="${pkgDirBase}/archlinuxewe/os"
pkgSrcDir="$(dirname $(readlink -f -n "$0"))"
[ "$(hostname)" == "rechenknecht" ] && \
  [ "${pkgSrcDir}" == "${HOME}" ] && \
    aufRechenknecht=true || \
    aufRechenknecht=false
branch="HEAD"
force=false
upload=true
updateMe=true
abortOnMakepkgError=true
cleanCache=false
cleanUnsigned=false
checkDbSig='-v'
checkMakepkg='--check'
rmAddPkgs=false
log=""
logFile=""

# print help screen
usage()
{
  >&2 echo ''
  >&2 echo 'Skript zum Updaten der selbsterstellten Archlinux-Pakete'
  >&2 echo ''
  >&2 echo 'Optionen:'
  >&2 echo '    -b|--branch [commitish]:'
  >&2 echo '      branch to check out - defaults to "HEAD"'
  >&2 echo '    --cleanCache:'
  >&2 echo '      clean up pacman cache in build environments'
  >&2 echo '    -c|--cleanUnsigned:'
  >&2 echo '      remove unsigned Packages first'
  >&2 echo '    -f|--force:'
  >&2 echo '      rebuild packages even if they look up to date'
  >&2 echo '    --ignore-wrong-db-signature:'
  >&2 echo '      do not verify old database signature'
  >&2 echo '    -L|--log [file]:'
  >&2 echo '      generate logfile of makepkg-runs'
  >&2 echo '    --no-abortOnMakepkgError:'
  >&2 echo '      do not abort when makepkg returns non-zero'
  >&2 echo '    --no-check:'
  >&2 echo '      pass --nocheck to makepkg'
  >&2 echo '    --no-selfUpdate:'
  >&2 echo '      do not update this script from git'
  >&2 echo '    --no-upload:'
  >&2 echo '      do not upload built packages from rechenknecht to jeti100'
  >&2 echo '    --only $package:'
  >&2 echo '      only consider $package for build and update process'
  >&2 echo '    -r|--removeAdditionalPackages:'
  >&2 echo '      remove additional packages from build environments'
  >&2 echo ''
  rm -rf "${tmpDir}"
  exit 1
}

# clean up before exit
cleanUp () {
  if [ -n "${log}" ]
  then
    [ -z "${logFile}" ] && logFile="/tmp/$(date '+%F-%H-%M-%S')"
    [[ "${logFile}" == *".tar.gz" ]] || logFile="${logFile}.tar.gz"
    cd "${tmpDir}"
    logFiles="$(
      for s in */*.log
      do
        [ -r "${s}" ] && echo "${s}"
      done | \
        tr "\n" " "
    )"
    if [ -n "${logFiles}" ]
    then
      tar -czf "${logFile}" ${logFiles}
      echo 'Logfiles saved to "'"${logFile}"'".'
    fi
  fi
  rm -rf "${tmpDir}"
}

echo 'parse arguments ...'
eval set -- "$(
  getopt -o b:cfL::r \
    --long branch: \
    --long cleanCache \
    --long cleanUnsigned \
    --long force \
    --long ignore-wrong-db-signature \
    --long log:: \
    --long no-abortOnMakepkgError \
    --long no-check \
    --long no-selfUpdate \
    --long no-upload \
    --long only: \
    --long removeAdditionalPackages \
    -n "$(basename "$0")" \
    -- "$@" \
    || echo "usage"
)"
args=("$@")

while true
do
  case $1 in
    -b|--branch)
      shift
      branch="$1"
      echo '"branch '"${branch}"'" erkannt'
    ;;
    --cleanCache)
      echo '"cleanCache" erkannt'
      cleanCache=true
    ;;
    -c|--cleanUnsigned)
      echo '"cleanUnsigned" erkannt'
      cleanUnsigned=true
    ;;
    -f|--force)
      echo '"force" erkannt'
      force=true
    ;;
    --ignore-wrong-db-signature)
      echo '"ignore-wrong-db-signature" erkannt'
      checkDbSig=''
    ;;
    -L|--log)
      shift
      echo '"log" erkannt'
      log='-L'
      logFile="$1"
    ;;
    --no-abortOnMakepkgError)
      echo '"no-abortOnMakepkgError" erkannt'
      abortOnMakepkgError=false
    ;;
    --no-check)
      echo '"no-check" erkannt'
      checkMakepkg='--nocheck'
    ;;
    --no-selfUpdate)
      echo '"no-selfUpdate" erkannt'
      updateMe=false
    ;;
    --no-upload)
      echo '"no-upload" erkannt'
      upload=false
    ;;
    --only)
      shift
      echo '"only" erkannt'
      onlyPackages[${#onlyPackages[@]}]="$1"
    ;;
    -r|--removeAdditionalPackages)
      echo '"removeAdditionalPackages" erkannt'
      rmAddPkgs=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo "FEHLER: Verstehe Option \"$1\" doch nicht! Ich beende."
      rm -rf "${tmpDir}"
      exit 1
  esac
  shift
done

if [ ! $# -eq 0 ]
then
  >&2 echo 'FEHLER: Zu viele Argumente! Ich beende.'
  usage
fi

echo '... done'

$aufRechenknecht && pkgSrcDir="${pkgSrcDir}/archPackages"
cd "${pkgSrcDir}"

# extract git ${branch}
git archive --format tar "${branch}" | \
  tar -x -C "${tmpDir}" -f -

cd "${tmpDir}"

# update $0
if ${updateMe} && ${aufRechenknecht} && [ -e "$(basename $0)" ] && ! diff -q "$(basename $0)" "$0" &> /dev/null
then
  cp "$(basename $0)" "$0"
  $0 "${args[@]}"
  err=$?
  rm -rf "${tmpDir}"
  exit ${err}
fi

# clean unsigned packages
if ${cleanUnsigned} && ${aufRechenknecht}
then
  ssh paule@127.0.0.1 '
    rmPaket() {
      [ "$(basename "$(pwd)")" == "any" ] || \
        repo-remove '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "${1%-*-*-*.pkg.tar.xz}"
      rm "$1"
      [ -e "$1.sig" ] && rm "$1.sig"
    }

    cd "'${pkgDir}'"
    for arch in *
    do
      [ -d "${arch}" ] || continue
      cd "${arch}"
      for paket in *.pkg.tar.xz
      do
        [ -e "${paket}.sig" ] && continue
        rmPaket ${paket}
      done
      cd ..
    done
  '
fi

unset pakete
unset archs
unset verss

# collect packages
for paket in $( \
  echo */PKGBUILD | \
    sed "s|/PKGBUILD||g" \
  )
do
  [ -d "${paket}" ] || continue
  [ ${#onlyPackages[@]} -eq 0 ] && consider=true || consider=false
  for ((i=0; i<${#onlyPackages[@]}; i++))
  do
    [ "${onlyPackages[${i}]}" == "${paket}" ] && consider=true
  done
  ${consider} || continue

  . ${paket}/PKGBUILD

  for singleArch in "${arch[@]}"
  do

    [ "${singleArch}" == "x86_64" ] || \
      [ "${singleArch}" == "i686" ] || \
      [ "${singleArch}" == "any" ] || \
      continue

    pkgFile="${paket}-${pkgver}-${pkgrel}-${singleArch}.pkg.tar.xz"

    allExist=true
    [ "${singleArch}" == "any" ] && \
      for dir in ${pkgDir}/*
      do
        [ "${dir}" == "any" ] && continue
        [ ! -d "${dir}" ] && continue
        [ ! -e "${dir}/${pkgFile}" ] && allExist=false
      done

    if ${force} || \
      ! ${aufRechenknecht} || \
      [ ! -e "${pkgDir}/${singleArch}/${pkgFile}" ] || \
      ! ${allExist}
    then
      pakete[${#pakete[@]}]="${paket}"
      archs[${#archs[@]}]="${singleArch}"
      verss[${#verss[@]}]="${pkgver}-${pkgrel}"
    fi
  done
done

if ${aufRechenknecht}
then

# check sanity of build-chroots
  err=false
  for chrootArch in 32 64
  do
    [ -d "/opt/arch${chrootArch}" ] || continue
    basePakete="$(
      echo '
        pacman -Qg base base-devel | \
          cut -d " " -f 2 | \
          sort -u
        exit
      ' | \
        chrootUnpriv "${chrootArch}"
    )"
    explizitinstalliertePakete="$(
      echo '
        pacman -Qtt | \
          cut -d " " -f 1
        exit
      ' | \
        chrootUnpriv "${chrootArch}"
    )"
    alleinstalliertenPakete="$(
      echo '
        pacman -Q | \
          cut -d " " -f 1
        exit
      ' | \
        chrootUnpriv "${chrootArch}"
    )"
    zuvielePakete="$(
      (
        (
          (
            echo "${explizitinstalliertePakete}"
            echo "${basePakete}"
          ) | \
            sort | \
            uniq -d
          echo "${explizitinstalliertePakete}"
        ) | \
          sort | \
          uniq -u
        echo '
          pacman -Qdt | \
            cut -d " " -f 1
          exit
        ' | \
          chrootUnpriv "${chrootArch}"
      ) | \
        sort -u
    )"
    zuwenigePakete="$(
      (
        (
          echo "${alleinstalliertenPakete}"
          echo "${basePakete}"
        ) | \
          sort | \
          uniq -d
        echo "${basePakete}"
      ) | \
        sort | \
        uniq -u
    )"

    if [ -n "${zuvielePakete}" ]
    then
      >&2 echo 'zu viele Pakete in "'/opt/arch${chrootArch}'" installiert:'
      >&2 echo "${zuvielePakete}"
      if ${rmAddPkgs}
      then
        echo '
          echo "'"${zuvielePakete}"'" | \
            xargs -n1 sudo pacman -Rs --noconfirm
          exit
        ' | \
          chrootUnpriv "${chrootArch}"
      else
        err=true
      fi
    fi
    if [ -n "${zuwenigePakete}" ]
    then
      >&2 echo 'zu wenige Pakete in "'/opt/arch${chrootArch}'" installiert:'
      >&2 echo "${zuwenigePakete}"
      err=true
    fi
  done

  if ${err}
  then
    cleanUp
    exit 1
  fi

  maxErr=0

# generate current makepkg.conf and makepkg32.conf
  cd "${tmpDir}"
  cp /etc/makepkg.conf .
  patch -p0 -i makepkgSystem.patch
  sed 's|^\(BUILDENV=.*\)!sign|\1sign|' -i makepkg.conf
  cp makepkg{,32}.conf
  patch -p0 -i makepkg32.patch

# build packages
  for ((i=0; i<${#pakete[@]}; i++))
  do
    paket="${pakete[${i}]}"
    arch="${archs[${i}]}"
    vers="${verss[${i}]}"

    cd "${tmpDir}/${paket}"
    rm -f ${paket}-*-${arch}.pkg.tar.xz*
    rm -rf src pkg

    if [ "${arch}" == "i686" ]
    then
      # 32-bit is built in chroot /opt/arch32
      (
        ${cleanCache} && \
          echo 'sudo pacman -Sc --noconfirm && \'
        echo '\
          sudo pacman -Sy && \
          cd "'"${tmpDir}/${paket}"'" && \
          _targetHost='\''--host=i686-pc-linux-gnu'\'' makepkg '${log} ${checkMakepkg}' --config "'"${tmpDir}"'/makepkg32.conf" -fsr --noconfirm
          exit $?
        '
      ) | \
        chrootUnpriv 32
      err[${i}]=$?
    else
      # 64-bit is built in chroot /opt/arch64
      (
        ${cleanCache} && \
          echo 'sudo pacman -Sc --noconfirm && \'
        echo '\
          sudo pacman -Sy && \
          cd "'"${tmpDir}/${paket}"'" && \
          makepkg '${log} ${checkMakepkg}' --config "'"${tmpDir}"'/makepkg.conf" -fsr --noconfirm
          exit $?
        '
      ) | \
        chrootUnpriv 64
      err[${i}]=$?
    fi
    if ${abortOnMakepkgError} && [ ${err[${i}]} -ne 0 ]
    then
      cleanUp
      exit ${err[${i}]}
    fi
    [ ${err[${i}]} -gt ${maxErr} ] && maxErr=${err[${i}]}

    if ! gpg --verify ${paket}-${vers}-${arch}.pkg.tar.xz{.sig,} &> /dev/null
    then
      key="$(
        grep '^GPGKEY=' "${tmpDir}/makepkg.conf" | \
          sed 's|^GPGKEY=\s*"\(.*\)"\s*$|0x\1|'
      )"
      rm -f "${paket}-${vers}-${arch}.pkg.tar.xz.sig"
      gpg --detach-sign -u "${key}" --no-armor ${paket}-${vers}-${arch}.pkg.tar.xz
    fi

    if ${upload} && [ ${err[${i}]} -eq 0 ]
    then

# upload package and update db
      cd "${tmpDir}/${paket}"
      tar -cf - ${paket}-${vers}-${arch}.pkg.tar.xz{,.sig} |
        ssh paule@127.0.0.1 '

          rmPaket() {
            [ "$(basename "$(pwd)")" == "any" ] || \
              repo-remove '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "${1%-*-*-*.pkg.tar.xz}"
            rm "$1"
            [ -e "$1.sig" ] && rm "$1.sig"
          }

          addPaket() {
            neu="$1"
            for alt in $(
              ls -1 | \
                grep "^${neu%-*-*-*.pkg.tar.xz}-[^-]\+-[^-]\+-'${arch}'\.pkg\.tar\.xz\$"
            )
            do
              [ "${alt}" == "${neu}" ] && continue
              rmPaket "${alt}"
            done
            [ "$(basename "$(pwd)")" == "any" ] || \
              repo-add '${checkDbSig}' -s -k 0x20194BA1 archlinuxewe.db.tar.gz "${neu}"
          }

          cd '${pkgDir}/${arch}/'
          if [ -e "'${paket}-${vers}-${arch}.pkg.tar.xz'" ]
          then
            rmPaket "'${paket}-${vers}-${arch}.pkg.tar.xz'"
          fi
          tar -xf -
          addPaket "'${paket}-${vers}-${arch}.pkg.tar.xz'"

          if [ "'${arch}'" == "any" ]
          then
            cd ..
            for lArch in *
            do
              [ ! -d "${lArch}" ] && continue
              [ "${lArch}" == "any" ] && continue
              cd "${lArch}"

              if [ -e "'${paket}-${vers}-${arch}.pkg.tar.xz'" ]
              then
                rmPaket "'${paket}-${vers}-${arch}.pkg.tar.xz'"
              fi
              ln -s "'${pkgDir}/any/${paket}-${vers}-${arch}.pkg.tar.xz'" "'${paket}-${vers}-${arch}.pkg.tar.xz'"
              if [ -e "'${pkgDir}/any/${paket}-${vers}-${arch}.pkg.tar.xz.sig'" ]
              then
                ln -s "'${pkgDir}/any/${paket}-${vers}-${arch}.pkg.tar.xz.sig'" "'${paket}-${vers}-${arch}.pkg.tar.xz.sig'"
              fi
              addPaket "'${paket}-${vers}-${arch}.pkg.tar.xz'"

              cd ..
            done
          fi
        '
    fi
  done

# check db
  if ${upload}
  then
    ssh paule@127.0.0.1 '
      cd '${pkgDir}'
      err=0

      for arch in *
      do
        [ ! -d "${arch}" ] && continue
        [ "${arch}" == "any" ] && continue

        for paketFehler in $( \
          (
            tar -Oxzf '${pkgDir}'/${arch}/archlinuxewe.db.tar.gz | \
              grep -A1 "^%FILENAME%\$" | \
              grep -v "^%FILENAME%\$" | \
              grep -v -- "^--\$"
            ls '${pkgDir}'/${arch} | \
              grep "^.*\.pkg\.tar\.xz\$"
          ) | \
            sort | \
            uniq -u
        )
        do

          err=1
          if [ -e "'${pkgDir}'/${arch}/${paketFehler}" ]
          then
            >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur im Verzeichnisbaum vorhanden."
          else
            >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur in der Datenbank vorhanden."
          fi

        done

        for signaturWarnungen in $( \
          ls '${pkgDir}'/${arch} | \
            grep "^.*\.pkg\.tar\.xz\(\.sig\)\?\$" | \
            sed "s|\.sig\$||" | \
            sort | \
            uniq -u
        )
        do

          if [ -e "'${pkgDir}'/${arch}/${signaturWarnungen}" ]
          then
            >&2 echo "WARNUNG: Paket '${pkgDir}'/${arch}/${signaturWarnungen} hat keine Signatur."
          else
            err=1
            >&2 echo "FEHLER: Zur Signatur '${pkgDir}'/${arch}/${signaturWarnungen}.sig gibt es kein Paket."
          fi

        done

      done

      exit ${err}
    '
  fi

  cleanUp
  exit ${maxErr}

else

  namcap */PKGBUILD

# check packages
  err=false
  for ((i=0; i<${#pakete[@]}; i++))
  do
    paket="${pakete[${i}]}"

    . ${paket}/PKGBUILD

    if echo "${source}" | grep -q "://opensources\.eckner\.net/"
    then

# one of Erich's packages

      if grep -q "sha512sums" ${paket}/PKGBUILD
      then
        sumTyp="sha512sum"
        pkgSum="${sha512sums[@]}"
      elif grep -q "sha256sums" ${paket}/PKGBUILD
      then
        sumTyp="sha256sum"
        pkgSum="${sha256sums[@]}"
      fi
      remSum="$(curl -o - "${source#*::}&post=${sumTyp}" 2> /dev/null)"
      if [ ! "${remSum}" == "${pkgSum}" ]
      then
        >&2 echo "${sumTyp}s von ${pkgname} stimmt nicht: ${remSum} vs. ${pkgSum}."
        err=true
      fi

    elif echo "${source}" | grep -q "://arch\.eckner\.net/"
    then

# a modified official package

      offVer="$( \
        ssh paule@jeti100 'find '"${pkgDirBase}"' -type f -name "'"${paket}"'-*.pkg.tar.xz" -exec basename "{}" \; ' | \
          sort -V | \
          tail -n1 \
        )"
      if [ ! "${offVer}" == "${pkgname}-${pkgver}-${pkgrel}-${arch}.pkg.tar.xz" ]
      then
        >&2 echo "${pkgname} ist in den offiziellen Quellen aktueller: ${offVer}."
        err=true
      fi

    fi

  done
  if ${err}
  then
    rm -rf "${tmpDir}"
    exit 1
  fi

  cd "${pkgSrcDir}"
  git push
  err=$?
  if [ ${err} -ne 0 ]
  then
    rm -rf "${tmpDir}"
    exit ${err}
  fi

  ssh -p 22222 141.35.51.219 "~/archPackagesUpdate" "${args[@]}"
  err=$?
  if [ ${err} -ne 0 ]
  then
    rm -rf "${tmpDir}"
    exit ${err}
  fi

# clean up
  rm -rf "${tmpDir}"

fi