#!/bin/bash # init variables tmpDir="$(mktemp -d /tmp/archive.XXXXXX)" trap 'rm -rf --one-file-system "${tmpDir}"' EXIT pkgDirBase="/srv/arch-mirror/arch/arch" pkgDir="${pkgDirBase}/archlinuxewe/os" pkgDir32="${pkgDirBase}/archlinux32/x86_64/releng" pkgSrcDir="$(dirname $(readlink -f -n "$0"))" [ "$(hostname)" == "rechenknecht" ] && \ [ "${pkgSrcDir}" == "${HOME}" ] && \ aufRechenknecht=true || \ aufRechenknecht=false branch="HEAD" force=false upload=true updateMe=true abortOnMakepkgError=true cleanCache=false cleanUnsigned=false checkDbSig='-v' checkMakepkg='--check' checkSanity=true rmAddPkgs=false log='' logFile='-' printOnly=false # print help screen usage() { >&2 echo '' >&2 echo 'Skript zum Updaten der selbsterstellten Archlinux-Pakete' >&2 echo '' >&2 echo 'Optionen:' >&2 echo ' -b|--branch [commitish]:' >&2 echo ' branch to check out - defaults to "HEAD"' >&2 echo ' --cleanCache:' >&2 echo ' clean up pacman cache in build environments' >&2 echo ' -c|--cleanUnsigned:' >&2 echo ' remove unsigned Packages first' >&2 echo ' -f|--force:' >&2 echo ' rebuild packages even if they look up to date' >&2 echo ' --ignore-wrong-db-signature:' >&2 echo ' do not verify old database signature' >&2 echo ' -L|--log file:' >&2 echo ' generate logfile of makepkg-runs ("-" as file autogenerates name)' >&2 echo ' --no-abortOnMakepkgError:' >&2 echo ' do not abort when makepkg returns non-zero' >&2 echo ' --no-check:' >&2 echo ' pass --nocheck to makepkg' >&2 echo ' --no-sanity-check:' >&2 echo ' skip check for sanity of packages' >&2 echo ' --no-selfUpdate:' >&2 echo ' do not update this script from git' >&2 echo ' --no-upload:' >&2 echo ' do not upload built packages from rechenknecht to jeti100' >&2 echo ' --only $package:' >&2 echo ' only consider $package for build and update process' >&2 echo ' -p|--print-only:' >&2 echo ' only print what would be done' >&2 echo ' -r|--removeAdditionalPackages:' >&2 echo ' remove additional packages from build environments' >&2 echo '' exit 1 } # clean up before exit cleanUp () { if [ -n "${log}" ] then [ "a${logFile}" == 'a-' ] && logFile="/tmp/$(date '+%F-%H-%M-%S')" [[ "${logFile}" == *".tar.gz" ]] || logFile="${logFile}.tar.gz" cd "${tmpDir}" logFiles="$( for s in */*.log do [ -r "${s}" ] && echo "${s}" done )" if [ -n "${logFiles}" ] then tar -czf "${logFile}" ${logFiles} echo 'Logfiles saved to "'"${logFile}"'".' fi fi } echo 'parse arguments ...' eval set -- "$( getopt -o b:cfL:pr \ --long branch: \ --long cleanCache \ --long cleanUnsigned \ --long force \ --long ignore-wrong-db-signature \ --long log: \ --long no-abortOnMakepkgError \ --long no-check \ --long no-sanity-check \ --long no-selfUpdate \ --long no-upload \ --long only: \ --long print-only \ --long removeAdditionalPackages \ -n "$(basename "$0")" \ -- "$@" \ || echo "usage" )" args=("$@") while true do case $1 in -b|--branch) shift branch="$1" echo '"branch '"${branch}"'" erkannt' ;; --cleanCache) echo '"cleanCache" erkannt' cleanCache=true ;; -c|--cleanUnsigned) echo '"cleanUnsigned" erkannt' cleanUnsigned=true ;; -f|--force) echo '"force" erkannt' force=true ;; --ignore-wrong-db-signature) echo '"ignore-wrong-db-signature" erkannt' checkDbSig='' ;; -L|--log) shift log='-L' logFile="$1" echo '"log '"${logFile}"'" erkannt' ;; --no-abortOnMakepkgError) echo '"no-abortOnMakepkgError" erkannt' abortOnMakepkgError=false ;; --no-check) echo '"no-check" erkannt' checkMakepkg='--nocheck' ;; --no-sanity-check) echo '"no-sanity-check" erkannt' checkSanity=false ;; --no-selfUpdate) echo '"no-selfUpdate" erkannt' updateMe=false ;; --no-upload) echo '"no-upload" erkannt' upload=false ;; --only) shift echo '"only '"$1"'" erkannt' onlyPackages[${#onlyPackages[@]}]="$1" ;; -p|--print-only) echo '"print-only" erkannt' printOnly=true ;; -r|--removeAdditionalPackages) echo '"removeAdditionalPackages" erkannt' rmAddPkgs=true ;; --) shift break ;; *) >&2 echo "FEHLER: Verstehe Option \"$1\" doch nicht! Ich beende." exit 1 esac shift done if [ ! $# -eq 0 ] then >&2 echo 'FEHLER: Zu viele ('$#') Argumente:' for ((i=1; i<=$#; i++)) do >&2 echo "'${!i}'" done >&2 echo 'Ich beende.' usage fi if ! ${checkSanity} then if ${upload} || ${updateMe} then >&2 echo 'FEHLER: "--no-sanity-check" geht nur zusammen mit "--no-upload" und "--no-selfUpdate"! Ich beende.' exit 1 fi fi echo '... done' ${aufRechenknecht} && pkgSrcDir="${pkgSrcDir}/archPackages" cd "${pkgSrcDir}" # extract git ${branch} git archive --format tar "${branch}" | \ tar -x -C "${tmpDir}" -f - cd "${tmpDir}" if [ -f '.gitmodules' ]; then sed -n ' /^\s*path = /{ N s/^\s*path = // s/\n\s*url = / / s,ssh://aur@aur.archlinux.org/,https://aur.archlinux.org/, p } ' '.gitmodules' | \ while read -r path url; do if [ -n "$(ls -A "${path}")" ]; then git submodule update -- "${path}" continue fi mkdir -p "${path}" git clone "${url}" "${path}" done fi # update $0 if ${updateMe} && ${aufRechenknecht} && [ -e "$(basename "$0")" ] && ! diff -q "$(basename "$0")" "$0" &> /dev/null then cp "$(basename "$0")" "$0" "$0" "${args[@]}" err=$? exit ${err} fi # clean unsigned packages if ${cleanUnsigned} && ${aufRechenknecht} then ssh paule@127.0.0.1 "$( printf ' rmPaket() { if [ "$(basename "$(pwd)")" != "any" ] then if [ "$(basename "$(pwd)")" == "releng" ] then repo-remove releng.db.tar.gz "${1%%-*-*-*.pkg.tar.xz}" else repo-remove %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "${1%%-*-*-*.pkg.tar.xz}"' \ "${checkDbSig}" printf ' fi fi rm "$1" [ -e "$1.sig" ] && rm "$1.sig" }' printf ' for arch in %s/* %s' \ "${pkgDir}" "${pkgDir32}" printf ' do [ -d "${arch}" ] || continue cd "${arch}" for paket in *.pkg.tar.xz do [ -e "${paket}.sig" ] && continue rmPaket "${paket}" done done' )" fi unset pakete unset pakete_teile unset archs unset verss unset repos if ${aufRechenknecht} then # collect packages for paket in $( \ echo */PKGBUILD | \ sed "s|/PKGBUILD||g" \ ) do [ -d "${paket}" ] || continue [ ${#onlyPackages[@]} -eq 0 ] && consider=true || consider=false for ((i=0; i<${#onlyPackages[@]}; i++)) do [[ "${paket}" == "${onlyPackages[${i}]}" ]] && consider=true done ${consider} || continue unset epoch unset pkgname . "${paket}/PKGBUILD" if grep -q '#\s*repo:\s*releng\s*\(#.*\)\?$' "${paket}/PKGBUILD"; then repo='releng' else repo='archlinuxewe' fi for singleArch in "${arch[@]}"; do [ "${singleArch}" == "x86_64" ] || \ [ "${singleArch}" == "i686" ] || \ [ "${singleArch}" == "any" ] || \ continue allExist=true for pkgnam in "${pkgname[@]}"; do pkgFile="${pkgnam}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${singleArch}.pkg.tar.xz" if [ "${repo}" == 'releng' ]; then [ ! -e "${pkgDir32}/${pkgFile}" ] && allExist=false else [ "${singleArch}" == "any" ] && \ for dir in ${pkgDir}/* do [ ! -d "${dir}" ] && continue [ ! -e "${dir}/${pkgFile}" ] && allExist=false done if [ ! -e "${pkgDir}/${singleArch}/${pkgFile}" ]; then allExist=false fi fi done if ${force} || \ ! ${allExist} then pakete[${#pakete[@]}]="${paket}" pakete_teile[${#pakete_teile[@]}]="${pkgname[@]}" archs[${#archs[@]}]="${singleArch}" verss[${#verss[@]}]="${epoch:+${epoch}:}${pkgver}-${pkgrel}" repos[${#repos[@]}]="${repo}" fi done done # only print packages to be built? if ${printOnly}; then printf '%d scheduled builds:\n' "${#pakete[@]}" for ((i=0; i<${#pakete[@]}; i++)); do echo " ${pakete[${i}]} ${verss[${i}]} ${archs[${i}]}" printf ' %s\n' ${pakete_teile[${i}]} done exit 0 fi # check sanity of build-chroots err=false for chrootArch in 32 64 do [ -d "/opt/arch${chrootArch}" ] || continue basePakete=$( ( echo ' pacman -Qg base base-devel | \ cut -d " " -f 2 | \ sort -u exit ' | \ chrootUnpriv "${chrootArch}" printf '%s\n' \ 'update-all' 'ccache' 'git' 'mercurial' 'bzr' 'subversion' ) | \ sort -u ) explizitinstalliertePakete=$( echo ' pacman -Qtt | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" | \ sed ' /^msmtp$/d /^report-installed-packages$/d /^sendmailadvanced$/d ' ) alleinstalliertenPakete=$( echo ' pacman -Q | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" ) zuvielePakete=$( ( ( ( echo "${explizitinstalliertePakete}" echo "${basePakete}" ) | \ sort | \ uniq -d echo "${explizitinstalliertePakete}" ) | \ sort | \ uniq -u echo ' pacman -Qdt | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" ) | \ sort -u ) zuwenigePakete=$( ( ( echo "${alleinstalliertenPakete}" echo "${basePakete}" ) | \ sort | \ uniq -d echo "${basePakete}" ) | \ sort | \ uniq -u ) if [ -n "${zuvielePakete}" ] then >&2 echo 'zu viele Pakete in "'"/opt/arch${chrootArch}"'" installiert:' >&2 echo "${zuvielePakete}" if ${rmAddPkgs} then echo ' echo "'"${zuvielePakete}"'" | \ xargs -n1 sudo pacman -Rs --noconfirm exit ' | \ chrootUnpriv "${chrootArch}" else err=true fi fi if [ -n "${zuwenigePakete}" ] then >&2 echo 'zu wenige Pakete in "'"/opt/arch${chrootArch}"'" installiert:' >&2 echo "${zuwenigePakete}" err=true fi done if ${err} then cleanUp exit 1 fi maxErr=0 # generate current makepkg.conf and makepkg32.conf cd "${tmpDir}" cp /etc/makepkg.conf . if ! patch -p0 -i makepkgSystem.patch then cleanUp exit 1 fi sed 's|^\(BUILDENV=.*\)!sign|\1sign|' -i makepkg.conf cp makepkg{,32}.conf if ! patch -p0 -i makepkg32.patch then cleanUp exit 1 fi eval "$( grep '^SRCDEST=' makepkg.conf )" if [ -n "${SRCDEST}" ]; then # update git sources find "${SRCDEST}" -mindepth 2 -maxdepth 2 -name 'HEAD' | \ sed 's|/[^/]*$||' | \ while read -r repo; do git -C "${repo}" fetch --all -p done # update svn sources find "${SRCDEST}" -mindepth 2 -maxdepth 2 -name '.svn' | \ sed 's|/[^/]*$||' | \ while read -r repo; do svn update "${repo}" done fi # build packages for ((i=0; i<${#pakete[@]}; i++)) do paket="${pakete[${i}]}" paket_teile=(${pakete_teile[${i}]}) arch="${archs[${i}]}" vers="${verss[${i}]}" repo="${repos[${i}]}" cd "${tmpDir}/${paket}" rm -f ${paket}-*-${arch}.pkg.tar.xz* rm -rf --one-file-system src pkg # report installed packages at end of package() function - if the build process succeeded, # these packages should work echo ' if [ "$(uname -m)" = "i686" ]; then eval "$( declare -f package | \ sed '"'"' $ i sudo report-installed-packages archlinux-reports@eckner.net buildmaster@archlinux32.org '"'"' )" fi ' >> PKGBUILD if [ "${arch}" = 'i686' ]; then # 32-bit is built in chroot /opt/arch32 ( ${cleanCache} && \ echo 'sudo pacman -Sc --noconfirm && \' echo '\ sudo pacman -Sy && \ cd "'"${tmpDir}/${paket}"'" && \ _targetHost='"'"'--host=i686-pc-linux-gnu'"'"' makepkg '"${log} ${checkMakepkg}"' --holdver --config "'"${tmpDir}"'/makepkg32.conf" -fsr --noconfirm --asdeps exit $? ' ) | \ chrootUnpriv 32 err[${i}]=$? elif [ "${arch}" = 'x86_64' ] || [ "${arch}" = 'any' ]; then # 64-bit and any is built in chroot /opt/arch64 ( ${cleanCache} && \ echo 'sudo pacman -Sc --noconfirm && \' echo '\ sudo pacman -Sy && \ cd "'"${tmpDir}/${paket}"'" && \ makepkg '"${log} ${checkMakepkg}"' --holdver --config "'"${tmpDir}"'/makepkg.conf" -fsr --noconfirm --asdeps exit $? ' ) | \ chrootUnpriv 64 err[${i}]=$? else printf 'unbekannte Architektur "%s"\n' "${arch}" err[${i}]=128 fi for teil in "${paket_teile[@]}"; do if [ ${err[${i}]} -ne 0 ]; then break fi namcap "${tmpDir}/${paket}/${teil}-${vers}-${arch}.pkg.tar.xz" > \ "${tmpDir}/namcap" err[${i}]=$? if [ -n "${log}" ] then cat "${tmpDir}/namcap" > \ "${tmpDir}/${paket}/${teil}-namcap-${arch}.log" fi if [ ${err[${i}]} -ne 0 ]; then break fi if grep "$( { printf '%s\\|' \ "E: ELF file (.*) found in an ['\"]any['\"] package" if ! grep -qF " ${teil} ist absichtlich nicht any ohne ELF Dateien" PKGBUILD; then printf '%s\\|' \ "W: No ELF files and not an ['\"]any['\"] package" fi } | \ sed 's@\\|$@@' )" "${tmpDir}/namcap"; then err[${i}]=64 fi done rm -f "${tmpDir}/namcap" if ${abortOnMakepkgError} && [ ${err[${i}]} -ne 0 ] then cleanUp exit ${err[${i}]} fi [ ${err[${i}]} -gt ${maxErr} ] && maxErr=${err[${i}]} eval "$( sed -n ' s/^GPGKEY=/key=/ T p ' "${tmpDir}/makepkg.conf" )" for teil in "${paket_teile[@]}"; do if ! gpg --verify ${teil}-${vers}-${arch}.pkg.tar.xz{.sig,} &> /dev/null; then rm -f "${teil}-${vers}-${arch}.pkg.tar.xz.sig" gpg --detach-sign -u "${key}" --no-armor ${teil}-${vers}-${arch}.pkg.tar.xz fi done if ${upload} && [ ${err[${i}]} -eq 0 ] then # upload package and update db cd "${tmpDir}/${paket}" tar -cf - $( printf " %s-${vers}-${arch}.pkg.tar.xz" "${paket_teile[@]}" printf " %s-${vers}-${arch}.pkg.tar.xz.sig" "${paket_teile[@]}" ) | \ ssh paule@127.0.0.1 "$( printf ' entferneAltePakete() { altePakete=$( ls -1 | \ grep "^$1-[^-]\+-[^-]\+-$2\.pkg\.tar\.xz\$" ) if [ -n "${altePakete}" ] then if [ "$(basename "$(pwd)")" != "any" ] then if [ "$(basename "$(pwd)")" == "releng" ] then repo-remove releng.db.tar.gz "$1" else repo-remove %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1"' \ "${checkDbSig}" printf ' fi fi for altesPaket in ${altePakete} do rm "${altesPaket}" if [ -e "${altesPaket}.sig" ] || [ -h "${altesPaket}.sig" ] then rm "${altesPaket}.sig" fi done fi }' printf ' addPaket() { if [ "$(basename "$(pwd)")" == "releng" ] then repo-add releng.db.tar.gz "$1" else repo-add %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1" fi }' \ "${checkDbSig}" if [ "${repo}" = 'releng' ] then printf ' cd "%s"' \ "${pkgDir32}" else printf ' cd "%s/%s"' \ "${pkgDir}" "${arch}" fi printf ' for teil in %s; do' \ "${paket_teile[*]}" printf ' entferneAltePakete "${teil}" "%s"' \ "${arch}" printf ' done' printf ' tar -xf - for teil in %s; do' \ "${paket_teile[*]}" if [ "${arch}" == "any" ] && [ "${repo}" != 'releng' ] then printf ' for lArch in $(ls ..) do [ ! -d "../${lArch}" ] && continue [ "${lArch}" == "any" ] && continue cd "../${lArch}" entferneAltePakete "${teil}" "any"' printf ' ln -s "%s/any/${teil}-%s-%s.pkg.tar.xz" "${teil}-%s-%s.pkg.tar.xz"' \ "${pkgDir}" "${vers}" "${arch}" "${vers}" "${arch}" printf ' if [ -e "%s/any/${teil}-%s-%s.pkg.tar.xz.sig" ]' \ "${pkgDir}" "${vers}" "${arch}" printf ' then ln -s "%s/any/${teil}-%s-%s.pkg.tar.xz.sig" "${teil}-%s-%s.pkg.tar.xz.sig"' \ "${pkgDir}" "${vers}" "${arch}" "${vers}" "${arch}" printf ' fi addPaket "${teil}-%s-%s.pkg.tar.xz"' \ "${vers}" "${arch}" printf ' done' else printf ' addPaket "${teil}-%s-%s.pkg.tar.xz"' \ "${vers}" "${arch}" fi printf ' done' )" fi done # check db if ${upload} then ssh paule@127.0.0.1 "$( printf ' err=0 for arch in %s/* %s' \ "${pkgDir}" "${pkgDir32}" printf ' do [ ! -d "${arch}" ] && continue [ "${arch##*/}" == "any" ] && continue for paketFehler in $( \ ( if [ "$(basename "${arch}")" == 'releng' ] then tar -Oxzf "${arch}/releng.db.tar.gz" else tar -Oxzf "${arch}/archlinuxewe.db.tar.gz" fi | \ grep -A1 "^%%FILENAME%%\$" | \ grep -v "^%%FILENAME%%\$" | \ grep -v -- "^--\$" ls "${arch}" | \ grep "^.*\.pkg\.tar\.xz\$" ) | \ sort | \ uniq -u ) do err=1 if [ -e "${arch}/${paketFehler}" ] then >&2 echo "FEHLER: Paket ${arch}/${paketFehler} ist nur im Verzeichnisbaum vorhanden." else >&2 echo "FEHLER: Paket ${arch}/${paketFehler} ist nur in der Datenbank vorhanden." fi done for signaturWarnungen in $( \ ls "${arch}" | \ grep "^.*\.pkg\.tar\.xz\(\.sig\)\?\$" | \ sed "s|\.sig\$||" | \ sort | \ uniq -u ) do if [ -e "${arch}/${signaturWarnungen}" ] then >&2 echo "WARNUNG: Paket ${arch}/${signaturWarnungen} hat keine Signatur." else err=1 >&2 echo "FEHLER: Zur Signatur ${arch}/${signaturWarnungen}.sig gibt es kein Paket." fi done done exit ${err}' )" fi cleanUp exit ${maxErr} else if ${checkSanity} then # check packages err=false namcap */PKGBUILD || err=true for paket in * do [ -r "${paket}/PKGBUILD" ] || continue if ! grep -q "^# Maintainer\s*:\s\+Erich Eckner\s\+\$" "${paket}/PKGBUILD" || \ grep "^# Maintainer\s*:" "${paket}/PKGBUILD" | \ grep -vq "^# Maintainer\s*:\s\+Erich Eckner\s\+\$" then >&2 echo "${paket}/PKGBUILD hat falsche(n) Maintainer" if [ -d "${paket}/.git" ]; then continue fi err=true fi unset epoch . "${paket}/PKGBUILD" if echo "${source}" | \ grep -q "://opensources\.eckner\.net/" then # one of Erich's packages if grep -q "sha512sums" "${paket}/PKGBUILD" then sumTyp="sha512sum" pkgSum="${sha512sums[@]}" elif grep -q "sha256sums" "${paket}/PKGBUILD" then sumTyp="sha256sum" pkgSum="${sha256sums[@]}" fi remSum="$(curl "${source#*::}&post=${sumTyp}" 2> /dev/null)" if [ ! "${remSum}" == "${pkgSum}" ] then >&2 echo "${sumTyp}s von ${pkgname} stimmt nicht: ${remSum} vs. ${pkgSum}." err=true fi elif echo "${source}" | \ grep -q "://arch\.eckner\.net/" then # a modified official package offVer="$( \ { printf '%s-%s%s-%s-%s.pkg.tar.xz\n' \ "${pkgname}" \ "${epoch:+${epoch}:}" \ "${pkgver}" \ "${pkgrel}" \ "${arch}" ssh paule@jeti100 'find '"${pkgDirBase}"' \( -type d \( -name armv6h -o -name armv7h -o -name aarch64 -o -name arm -o -name archlinux32 \) -prune -false \) -o -type f -name "'"${paket}"'-*.pkg.tar.xz" -exec basename "{}" \; ' } | \ sort -V | \ tail -n1 \ )" if [ ! "${offVer}" == "${pkgname}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${arch}.pkg.tar.xz" ] then >&2 echo "${pkgname} ist in den offiziellen Quellen aktueller: ${offVer}." err=true fi fi done if ${err} then exit 1 fi fi cd "${pkgSrcDir}" git push err=$? if [ ${err} -ne 0 ] then exit ${err} fi ssh -p 22222 makepkg@141.35.51.219 "~/archPackagesUpdate" "${args[@]}" err=$? if [ ${err} -ne 0 ] then exit ${err} fi fi