#!/bin/bash # init variables tmpDir="$(mktemp -d /tmp/archive.XXXXXX)" trap 'rm -rf --one-file-system "${tmpDir}"' EXIT pkgDirBase="/srv/arch-mirror/arch/arch" pkgDir="${pkgDirBase}/archlinuxewe/os" pkgSrcDir="$(dirname $(readlink -f -n "$0"))" [ "$(hostname)" == "rechenknecht" ] && \ [ "${pkgSrcDir}" == "${HOME}" ] && \ aufRechenknecht=true || \ aufRechenknecht=false branch="HEAD" force=false upload=true updateMe=true abortOnMakepkgError=true cleanCache=false cleanUnsigned=false checkDbSig='-v' checkMakepkg='--check' checkSanity=true rmAddPkgs=false log='' logFile='-' printOnly=false # print help screen usage() { >&2 echo '' >&2 echo 'Skript zum Updaten der selbsterstellten Archlinux-Pakete' >&2 echo '' >&2 echo 'Optionen:' >&2 echo ' -b|--branch [commitish]:' >&2 echo ' branch to check out - defaults to "HEAD"' >&2 echo ' --cleanCache:' >&2 echo ' clean up pacman cache in build environments' >&2 echo ' -c|--cleanUnsigned:' >&2 echo ' remove unsigned Packages first' >&2 echo ' -f|--force:' >&2 echo ' rebuild packages even if they look up to date' >&2 echo ' --ignore-wrong-db-signature:' >&2 echo ' do not verify old database signature' >&2 echo ' -L|--log file:' >&2 echo ' generate logfile of makepkg-runs ("-" as file autogenerates name)' >&2 echo ' --no-abortOnMakepkgError:' >&2 echo ' do not abort when makepkg returns non-zero' >&2 echo ' --no-check:' >&2 echo ' pass --nocheck to makepkg' >&2 echo ' --no-sanity-check:' >&2 echo ' skip check for sanity of packages' >&2 echo ' --no-selfUpdate:' >&2 echo ' do not update this script from git' >&2 echo ' --no-upload:' >&2 echo ' do not upload built packages from rechenknecht to jeti100' >&2 echo ' --only $package:' >&2 echo ' only consider $package for build and update process' >&2 echo ' -p|--print-only:' >&2 echo ' only print what would be done' >&2 echo ' -r|--removeAdditionalPackages:' >&2 echo ' remove additional packages from build environments' >&2 echo '' exit 1 } # clean up before exit cleanUp () { if [ -n "${log}" ] then [ "a${logFile}" == 'a-' ] && logFile="/tmp/$(date '+%F-%H-%M-%S')" [[ "${logFile}" == *".tar.gz" ]] || logFile="${logFile}.tar.gz" cd "${tmpDir}" logFiles="$( for s in */*.log do [ -r "${s}" ] && echo "${s}" done | \ tr "\n" " " )" if [ -n "${logFiles}" ] then tar -czf "${logFile}" ${logFiles} echo 'Logfiles saved to "'"${logFile}"'".' fi fi } echo 'parse arguments ...' eval set -- "$( getopt -o b:cfL:pr \ --long branch: \ --long cleanCache \ --long cleanUnsigned \ --long force \ --long ignore-wrong-db-signature \ --long log: \ --long no-abortOnMakepkgError \ --long no-check \ --long no-sanity-check \ --long no-selfUpdate \ --long no-upload \ --long only: \ --long print-only \ --long removeAdditionalPackages \ -n "$(basename "$0")" \ -- "$@" \ || echo "usage" )" args=("$@") while true do case $1 in -b|--branch) shift branch="$1" echo '"branch '"${branch}"'" erkannt' ;; --cleanCache) echo '"cleanCache" erkannt' cleanCache=true ;; -c|--cleanUnsigned) echo '"cleanUnsigned" erkannt' cleanUnsigned=true ;; -f|--force) echo '"force" erkannt' force=true ;; --ignore-wrong-db-signature) echo '"ignore-wrong-db-signature" erkannt' checkDbSig='' ;; -L|--log) shift log='-L' logFile="$1" echo '"log '"${logFile}"'" erkannt' ;; --no-abortOnMakepkgError) echo '"no-abortOnMakepkgError" erkannt' abortOnMakepkgError=false ;; --no-check) echo '"no-check" erkannt' checkMakepkg='--nocheck' ;; --no-sanity-check) echo '"no-sanity-check" erkannt' checkSanity=false ;; --no-selfUpdate) echo '"no-selfUpdate" erkannt' updateMe=false ;; --no-upload) echo '"no-upload" erkannt' upload=false ;; --only) shift echo '"only '"$1"'" erkannt' onlyPackages[${#onlyPackages[@]}]="$1" ;; -p|--print-only) echo '"print-only" erkannt' printOnly=true ;; -r|--removeAdditionalPackages) echo '"removeAdditionalPackages" erkannt' rmAddPkgs=true ;; --) shift break ;; *) >&2 echo "FEHLER: Verstehe Option \"$1\" doch nicht! Ich beende." exit 1 esac shift done if [ ! $# -eq 0 ] then >&2 echo 'FEHLER: Zu viele ('$#') Argumente:' for ((i=1; i<=$#; i++)) do >&2 echo "'${!i}'" done >&2 echo 'Ich beende.' usage fi if ! ${checkSanity} then if ${upload} || ${updateMe} then >&2 echo 'FEHLER: "--no-sanity-check" geht nur zusammen mit "--no-upload" und "--no-selfUpdate"! Ich beende.' exit 1 fi fi echo '... done' $aufRechenknecht && pkgSrcDir="${pkgSrcDir}/archPackages" cd "${pkgSrcDir}" # extract git ${branch} git archive --format tar "${branch}" | \ tar -x -C "${tmpDir}" -f - cd "${tmpDir}" # update $0 if ${updateMe} && ${aufRechenknecht} && [ -e "$(basename $0)" ] && ! diff -q "$(basename $0)" "$0" &> /dev/null then cp "$(basename $0)" "$0" $0 "${args[@]}" err=$? exit ${err} fi # clean unsigned packages if ${cleanUnsigned} && ${aufRechenknecht} then ssh paule@127.0.0.1 ' rmPaket() { [ "$(basename "$(pwd)")" == "any" ] || \ repo-remove '${checkDbSig}' -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "${1%-*-*-*.pkg.tar.xz}" rm "$1" [ -e "$1.sig" ] && rm "$1.sig" } cd "'${pkgDir}'" for arch in * do [ -d "${arch}" ] || continue cd "${arch}" for paket in *.pkg.tar.xz do [ -e "${paket}.sig" ] && continue rmPaket ${paket} done cd .. done ' fi unset pakete unset pakete_teile unset archs unset verss if ${aufRechenknecht} then # collect packages for paket in $( \ echo */PKGBUILD | \ sed "s|/PKGBUILD||g" \ ) do [ -d "${paket}" ] || continue [ ${#onlyPackages[@]} -eq 0 ] && consider=true || consider=false for ((i=0; i<${#onlyPackages[@]}; i++)) do [[ "${paket}" == ${onlyPackages[${i}]} ]] && consider=true done ${consider} || continue unset epoch unset pkgname . ${paket}/PKGBUILD for singleArch in "${arch[@]}"; do [ "${singleArch}" == "x86_64" ] || \ [ "${singleArch}" == "i686" ] || \ [ "${singleArch}" == "any" ] || \ continue allExist=true for pkgnam in "${pkgname[@]}"; do pkgFile="${pkgnam}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${singleArch}.pkg.tar.xz" [ "${singleArch}" == "any" ] && \ for dir in ${pkgDir}/* do [ ! -d "${dir}" ] && continue [ ! -e "${dir}/${pkgFile}" ] && allExist=false done if [ ! -e "${pkgDir}/${singleArch}/${pkgFile}" ]; then allExist=false fi done if ${force} || \ ! ${aufRechenknecht} || \ ! ${allExist} then pakete[${#pakete[@]}]="${paket}" pakete_teile[${#pakete_teile[@]}]="${pkgname[@]}" archs[${#archs[@]}]="${singleArch}" verss[${#verss[@]}]="${epoch:+${epoch}:}${pkgver}-${pkgrel}" fi done done # only print packages to be built? if ${printOnly}; then printf '%d scheduled builds:\n' "${#pakete[@]}" for ((i=0; i<${#pakete[@]}; i++)); do echo " ${pakete[${i}]} ${verss[${i}]} ${archs[${i}]}" printf ' %s\n' ${pakete_teile[${i}]} done exit 0 fi # check sanity of build-chroots err=false for chrootArch in 32 64 do [ -d "/opt/arch${chrootArch}" ] || continue basePakete="$( ( echo ' pacman -Qg base base-devel | \ cut -d " " -f 2 | \ sort -u exit ' | \ chrootUnpriv "${chrootArch}" printf '%s\n' \ 'update-all' 'ccache' 'git' 'mercurial' 'bzr' 'subversion' ) | \ sort -u )" explizitinstalliertePakete="$( echo ' pacman -Qtt | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" | \ sed ' /^msmtp$/d /^report-installed-packages$/d /^sendmailadvanced$/d ' )" alleinstalliertenPakete="$( echo ' pacman -Q | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" )" zuvielePakete="$( ( ( ( echo "${explizitinstalliertePakete}" echo "${basePakete}" ) | \ sort | \ uniq -d echo "${explizitinstalliertePakete}" ) | \ sort | \ uniq -u echo ' pacman -Qdt | \ cut -d " " -f 1 exit ' | \ chrootUnpriv "${chrootArch}" ) | \ sort -u )" zuwenigePakete="$( ( ( echo "${alleinstalliertenPakete}" echo "${basePakete}" ) | \ sort | \ uniq -d echo "${basePakete}" ) | \ sort | \ uniq -u )" if [ -n "${zuvielePakete}" ] then >&2 echo 'zu viele Pakete in "'/opt/arch${chrootArch}'" installiert:' >&2 echo "${zuvielePakete}" if ${rmAddPkgs} then echo ' echo "'"${zuvielePakete}"'" | \ xargs -n1 sudo pacman -Rs --noconfirm exit ' | \ chrootUnpriv "${chrootArch}" else err=true fi fi if [ -n "${zuwenigePakete}" ] then >&2 echo 'zu wenige Pakete in "'/opt/arch${chrootArch}'" installiert:' >&2 echo "${zuwenigePakete}" err=true fi done if ${err} then cleanUp exit 1 fi maxErr=0 # generate current makepkg.conf and makepkg32.conf cd "${tmpDir}" cp /etc/makepkg.conf . if ! patch -p0 -i makepkgSystem.patch then cleanUp exit 1 fi sed 's|^\(BUILDENV=.*\)!sign|\1sign|' -i makepkg.conf cp makepkg{,32}.conf if ! patch -p0 -i makepkg32.patch then cleanUp exit 1 fi # update git sources find "$( grep '^SRCDEST=' makepkg.conf | \ cut -d= -f2 )" -mindepth 2 -maxdepth 2 -name 'HEAD' | \ sed 's|/[^/]*$||' | \ while read -r repo; do git -C "${repo}" fetch --all -p done # update svn sources find "$( grep '^SRCDEST=' makepkg.conf | \ cut -d= -f2 )" -mindepth 2 -maxdepth 2 -name '.svn' | \ sed 's|/[^/]*$||' | \ while read -r repo; do svn update "${repo}" done # build packages for ((i=0; i<${#pakete[@]}; i++)) do paket="${pakete[${i}]}" paket_teile=(${pakete_teile[${i}]}) arch="${archs[${i}]}" vers="${verss[${i}]}" cd "${tmpDir}/${paket}" rm -f ${paket}-*-${arch}.pkg.tar.xz* rm -rf --one-file-system src pkg # report installed packages at end of package() function - if the build process succeeded, # these packages should work echo ' if [ "$(uname -m)" = "i686" ]; then eval "$( declare -f package | \ sed '"'"' $ i sudo report-installed-packages archlinux-reports@eckner.net buildmaster@archlinux32.org '"'"' )" fi ' >> PKGBUILD if [ "${arch}" = 'i686' ]; then # 32-bit is built in chroot /opt/arch32 ( ${cleanCache} && \ echo 'sudo pacman -Sc --noconfirm && \' echo '\ sudo pacman -Sy && \ cd "'"${tmpDir}/${paket}"'" && \ _targetHost='\''--host=i686-pc-linux-gnu'\'' makepkg '${log} ${checkMakepkg}' --holdver --config "'"${tmpDir}"'/makepkg32.conf" -fsr --noconfirm --asdeps exit $? ' ) | \ chrootUnpriv 32 err[${i}]=$? elif [ "${arch}" = 'x86_64' ] || [ "${arch}" = 'any' ]; then # 64-bit and any is built in chroot /opt/arch64 ( ${cleanCache} && \ echo 'sudo pacman -Sc --noconfirm && \' echo '\ sudo pacman -Sy && \ cd "'"${tmpDir}/${paket}"'" && \ makepkg '${log} ${checkMakepkg}' --holdver --config "'"${tmpDir}"'/makepkg.conf" -fsr --noconfirm --asdeps exit $? ' ) | \ chrootUnpriv 64 err[${i}]=$? else printf 'unbekannte Architektur "%s"\n' "${arch}" err[${i}]=128 fi for teil in "${paket_teile[@]}"; do if [ ${err[${i}]} -ne 0 ]; then break fi namcap "${tmpDir}/${paket}/${teil}-${vers}-${arch}.pkg.tar.xz" > \ "${tmpDir}/namcap" err[${i}]=$? if [ -n "${log}" ] then cat "${tmpDir}/namcap" > \ "${tmpDir}/${paket}/${teil}-namcap-${arch}.log" fi if [ ${err[${i}]} -ne 0 ]; then break fi if grep "$( { printf '%s\\|' \ "E: ELF file (.*) found in an ['\"]any['\"] package" if ! grep -qF " ${teil} ist absichtlich nicht any ohne ELF Dateien" PKGBUILD; then printf '%s\\|' \ "W: No ELF files and not an ['\"]any['\"] package" fi } | \ sed 's@\\|$@@' )" "${tmpDir}/namcap"; then err[${i}]=64 fi done rm -f "${tmpDir}/namcap" if ${abortOnMakepkgError} && [ ${err[${i}]} -ne 0 ] then cleanUp exit ${err[${i}]} fi [ ${err[${i}]} -gt ${maxErr} ] && maxErr=${err[${i}]} key="$( grep '^GPGKEY=' "${tmpDir}/makepkg.conf" | \ sed 's|^GPGKEY=\s*"\(.*\)"\s*$|0x\1|' )" for teil in "${paket_teile[@]}"; do if ! gpg --verify ${teil}-${vers}-${arch}.pkg.tar.xz{.sig,} &> /dev/null; then rm -f "${teil}-${vers}-${arch}.pkg.tar.xz.sig" gpg --detach-sign -u "${key}" --no-armor ${teil}-${vers}-${arch}.pkg.tar.xz fi done if ${upload} && [ ${err[${i}]} -eq 0 ] then # upload package and update db cd "${tmpDir}/${paket}" tar -cf - $( printf " %s-${vers}-${arch}.pkg.tar.xz" "${paket_teile[@]}" printf " %s-${vers}-${arch}.pkg.tar.xz.sig" "${paket_teile[@]}" ) | ssh paule@127.0.0.1 ' entferneAltePakete() { altePakete=$( ls -1 | \ grep "^$1-[^-]\+-[^-]\+-$2\.pkg\.tar\.xz\$" ) if [ -n "${altePakete}" ] then [ "$(basename "$(pwd)")" == "any" ] || \ repo-remove '${checkDbSig}' -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1" for altesPaket in ${altePakete} do rm "${altesPaket}" if [ -e "${altesPaket}.sig" ] || [ -h "${altesPaket}.sig" ] then rm "${altesPaket}.sig" fi done fi } addPaket() { [ "$(basename "$(pwd)")" == "any" ] || \ repo-add '${checkDbSig}' -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1" } cd '${pkgDir}/${arch}/' for teil in '"${paket_teile[@]}"'; do entferneAltePakete "${teil}" "'${arch}'" done tar -xf - for teil in '"${paket_teile[@]}"'; do addPaket "${teil}-'"${vers}-${arch}"'.pkg.tar.xz" if [ "'${arch}'" == "any" ] then for lArch in $(ls ..) do [ ! -d "../${lArch}" ] && continue [ "${lArch}" == "any" ] && continue cd "../${lArch}" entferneAltePakete "${teil}" "'${arch}'" ln -s "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz" "${teil}-'${vers}-${arch}'.pkg.tar.xz" if [ -e "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz.sig" ] then ln -s "'${pkgDir}'/any/${teil}-'${vers}-${arch}'.pkg.tar.xz.sig" "${teil}-'${vers}-${arch}'.pkg.tar.xz.sig" fi addPaket "${teil}-'${vers}-${arch}'.pkg.tar.xz" done fi done ' fi done # check db if ${upload} then ssh paule@127.0.0.1 ' cd '${pkgDir}' err=0 for arch in * do [ ! -d "${arch}" ] && continue [ "${arch}" == "any" ] && continue for paketFehler in $( \ ( tar -Oxzf '${pkgDir}'/${arch}/archlinuxewe.db.tar.gz | \ grep -A1 "^%FILENAME%\$" | \ grep -v "^%FILENAME%\$" | \ grep -v -- "^--\$" ls '${pkgDir}'/${arch} | \ grep "^.*\.pkg\.tar\.xz\$" ) | \ sort | \ uniq -u ) do err=1 if [ -e "'${pkgDir}'/${arch}/${paketFehler}" ] then >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur im Verzeichnisbaum vorhanden." else >&2 echo "FEHLER: Paket '${pkgDir}'/${arch}/${paketFehler} ist nur in der Datenbank vorhanden." fi done for signaturWarnungen in $( \ ls '${pkgDir}'/${arch} | \ grep "^.*\.pkg\.tar\.xz\(\.sig\)\?\$" | \ sed "s|\.sig\$||" | \ sort | \ uniq -u ) do if [ -e "'${pkgDir}'/${arch}/${signaturWarnungen}" ] then >&2 echo "WARNUNG: Paket '${pkgDir}'/${arch}/${signaturWarnungen} hat keine Signatur." else err=1 >&2 echo "FEHLER: Zur Signatur '${pkgDir}'/${arch}/${signaturWarnungen}.sig gibt es kein Paket." fi done done exit ${err} ' fi cleanUp exit ${maxErr} else if ${checkSanity} then # check packages err=false namcap */PKGBUILD || err=true for paket in * do [ -r "${paket}/PKGBUILD" ] || continue if ! grep -q "^# Maintainer\s*:\s\+Erich Eckner\s\+\$" ${paket}/PKGBUILD || \ grep "^# Maintainer\s*:" ${paket}/PKGBUILD | grep -vq "^# Maintainer\s*:\s\+Erich Eckner\s\+\$" then >&2 echo "${paket}/PKGBUILD hat falsche(n) Maintainer" err=true fi unset epoch . ${paket}/PKGBUILD if echo "${source}" | grep -q "://opensources\.eckner\.net/" then # one of Erich's packages if grep -q "sha512sums" ${paket}/PKGBUILD then sumTyp="sha512sum" pkgSum="${sha512sums[@]}" elif grep -q "sha256sums" ${paket}/PKGBUILD then sumTyp="sha256sum" pkgSum="${sha256sums[@]}" fi remSum="$(curl "${source#*::}&post=${sumTyp}" 2> /dev/null)" if [ ! "${remSum}" == "${pkgSum}" ] then >&2 echo "${sumTyp}s von ${pkgname} stimmt nicht: ${remSum} vs. ${pkgSum}." err=true fi elif echo "${source}" | grep -q "://arch\.eckner\.net/" then # a modified official package offVer="$( \ { printf '%s-%s%s-%s-%s.pkg.tar.xz\n' \ "${pkgname}" \ "${epoch:+${epoch}:}" \ "${pkgver}" \ "${pkgrel}" \ "${arch}" ssh paule@jeti100 'find '"${pkgDirBase}"' \( -type d \( -name armv6h -o -name armv7h -o -name aarch64 -o -name arm -o -name archlinux32 \) -prune -false \) -o -type f -name "'"${paket}"'-*.pkg.tar.xz" -exec basename "{}" \; ' } | \ sort -V | \ tail -n1 \ )" if [ ! "${offVer}" == "${pkgname}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${arch}.pkg.tar.xz" ] then >&2 echo "${pkgname} ist in den offiziellen Quellen aktueller: ${offVer}." err=true fi fi done if ${err} then exit 1 fi fi cd "${pkgSrcDir}" git push err=$? if [ ${err} -ne 0 ] then exit ${err} fi ssh -p 22222 makepkg@141.35.51.219 "~/archPackagesUpdate" "${args[@]}" err=$? if [ ${err} -ne 0 ] then exit ${err} fi fi