#!/bin/bash # init variables tmpDir="$(mktemp -d /tmp/archive.XXXXXX)" trap 'rm -rf --one-file-system "${tmpDir}"' EXIT pkgDirBase="/srv/arch-mirror/arch/arch" pkgDir="${pkgDirBase}/archlinuxewe/os" pkgDir32="${pkgDirBase}/archlinux32/x86_64/releng" pkgSrcDir="$(dirname $(readlink -f -n "$0"))" [ "$(hostname)" == "rechenknecht" ] && \ [ "${pkgSrcDir}" == "${HOME}" ] && \ aufRechenknecht=true || \ aufRechenknecht=false branch="HEAD" force=false upload=true updateMe=true abortOnMakepkgError=true cleanChroot='' cleanUnsigned=false checkDbSig='-v' checkMakepkg='--check' checkSanity=true rmAddPkgs=false log='' logFile='-' printOnly=false # print help screen usage() { >&2 echo '' >&2 echo 'Skript zum Updaten der selbsterstellten Archlinux-Pakete' >&2 echo '' >&2 echo 'Optionen:' >&2 echo ' --arch $arch:' >&2 echo ' only compile for $arch' >&2 echo ' -b|--branch [commitish]:' >&2 echo ' branch to check out - defaults to "HEAD"' >&2 echo ' -c|--cleanChroot:' >&2 echo ' clean chroot before building' >&2 echo ' --cleanUnsigned:' >&2 echo ' remove unsigned Packages first' >&2 echo ' -f|--force:' >&2 echo ' rebuild packages even if they look up to date' >&2 echo ' --ignore-wrong-db-signature:' >&2 echo ' do not verify old database signature' >&2 echo ' -L|--log file:' >&2 echo ' generate logfile of makepkg-runs ("-" as file autogenerates name)' >&2 echo ' --no-abortOnMakepkgError:' >&2 echo ' do not abort when makepkg returns non-zero' >&2 echo ' --no-check:' >&2 echo ' pass --nocheck to makepkg' >&2 echo ' --no-sanity-check:' >&2 echo ' skip check for sanity of packages' >&2 echo ' --no-selfUpdate:' >&2 echo ' do not update this script from git' >&2 echo ' --no-upload:' >&2 echo ' do not upload built packages from rechenknecht to jeti100' >&2 echo ' --only $package:' >&2 echo ' only consider $package for build and update process' >&2 echo ' -p|--print-only:' >&2 echo ' only print what would be done' >&2 echo '' exit 1 } # clean up before exit cleanUp () { if [ -n "${log}" ]; then [ "a${logFile}" == 'a-' ] && logFile="/tmp/$(date '+%F-%H-%M-%S')" [[ "${logFile}" == *".tar.gz" ]] || logFile="${logFile}.tar.gz" cd "${tmpDir}" logFiles="$( for s in */*.log; do [ -r "${s}" ] && echo "${s}" done )" if [ -n "${logFiles}" ]; then tar -czf "${logFile}" ${logFiles} echo 'Logfiles saved to "'"${logFile}"'".' fi fi } echo 'parse arguments ...' eval set -- "$( getopt -o b:cfL:pr \ --long arch: \ --long branch: \ --long cleanChroot \ --long cleanUnsigned \ --long force \ --long ignore-wrong-db-signature \ --long log: \ --long no-abortOnMakepkgError \ --long no-check \ --long no-sanity-check \ --long no-selfUpdate \ --long no-upload \ --long only: \ --long print-only \ --long removeAdditionalPackages \ -n "$(basename "$0")" \ -- "$@" \ || echo "usage" )" args=("$@") while true; do case $1 in --arch) shift echo '"arch '"$1"'" erkannt' onlyArchs[${#onlyArchs[@]}]="$1" ;; -b|--branch) shift branch="$1" echo '"branch '"${branch}"'" erkannt' ;; -c|--cleanChroot) echo '"cleanChroot" erkannt' cleanChroot='-c' ;; --cleanUnsigned) echo '"cleanUnsigned" erkannt' cleanUnsigned=true ;; -f|--force) echo '"force" erkannt' force=true ;; --ignore-wrong-db-signature) echo '"ignore-wrong-db-signature" erkannt' checkDbSig='' ;; -L|--log) shift log='-L' logFile="$1" echo '"log '"${logFile}"'" erkannt' ;; --no-abortOnMakepkgError) echo '"no-abortOnMakepkgError" erkannt' abortOnMakepkgError=false ;; --no-check) echo '"no-check" erkannt' checkMakepkg='--nocheck' ;; --no-sanity-check) echo '"no-sanity-check" erkannt' checkSanity=false ;; --no-selfUpdate) echo '"no-selfUpdate" erkannt' updateMe=false ;; --no-upload) echo '"no-upload" erkannt' upload=false ;; --only) shift echo '"only '"$1"'" erkannt' onlyPackages[${#onlyPackages[@]}]="${1%/}" ;; -p|--print-only) echo '"print-only" erkannt' printOnly=true ;; -r|--removeAdditionalPackages) echo '"removeAdditionalPackages" erkannt' rmAddPkgs=true ;; --) shift break ;; *) >&2 echo "FEHLER: Verstehe Option \"$1\" doch nicht! Ich beende." exit 1 esac shift done if [ ! $# -eq 0 ]; then >&2 printf 'FEHLER: Zu viele (%s) Argumente:\n' "$#" >&2 printf '"%s"\n' "$@" >&2 echo 'Ich beende.' usage fi if ! ${checkSanity}; then if ${upload} || ${updateMe}; then >&2 echo 'FEHLER: "--no-sanity-check" geht nur zusammen mit "--no-upload" und "--no-selfUpdate"! Ich beende.' exit 1 fi fi echo '... done' ${aufRechenknecht} && pkgSrcDir="${pkgSrcDir}/archPackages" cd "${pkgSrcDir}" # extract git ${branch} git archive --format tar "${branch}" | \ tar -x -C "${tmpDir}" -f - cd "${tmpDir}" if [ -f '.gitmodules' ]; then sed -n ' /^\s*path = /{ N s/^\s*path = // s/\n\s*url = / / s,ssh://aur@aur.archlinux.org/,https://aur.archlinux.org/, p } ' '.gitmodules' | \ while read -r path url; do if [ -n "$(ls -A "${path}")" ]; then git submodule update -- "${path}" continue fi mkdir -p "${path}" git clone "${url}" "${path}" done fi # update $0 if ${updateMe} && ${aufRechenknecht} && [ -e "$(basename "$0")" ] && ! diff -q "$(basename "$0")" "$0" &> /dev/null; then cp "$(basename "$0")" "$0" "$0" "${args[@]}" err=$? exit ${err} fi # clean unsigned packages if ${cleanUnsigned} && ${aufRechenknecht}; then ssh paule@127.0.0.1 "$( printf ' rmPaket() { if [ "$(basename "$(pwd)")" != "any" ]; then if [ "$(basename "$(pwd)")" == "releng" ]; then repo-remove releng.db.tar.gz "${1%%-*-*-*.pkg.tar.xz}" else repo-remove %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "${1%%-*-*-*.pkg.tar.xz}"' \ "${checkDbSig}" printf ' fi fi rm "$1" [ -e "$1.sig" ] && rm "$1.sig" }' printf ' for arch in %s/* %s; do' \ "${pkgDir}" "${pkgDir32}" printf ' [ -d "${arch}" ] || continue cd "${arch}" for paket in *.pkg.tar.xz; do [ -e "${paket}.sig" ] && continue rmPaket "${paket}" done done' )" fi unset pakete unset pakete_teile_und_archs unset archs unset verss unset repos if ${aufRechenknecht}; then # collect packages for paket in $( \ echo */PKGBUILD | \ sed "s|/PKGBUILD||g" \ ); do [ -d "${paket}" ] || continue [ ${#onlyPackages[@]} -eq 0 ] && consider=true || consider=false for ((i=0; i<${#onlyPackages[@]}; i++)); do [[ "${paket}" == "${onlyPackages[${i}]%:*}" ]] && consider=true done ${consider} || continue unset epoch unset pkgname unset arch . "${paket}/PKGBUILD" if grep -q '#\s*repo:\s*releng\s*\(#.*\)\?$' "${paket}/PKGBUILD"; then repo='releng' else repo='archlinuxewe' fi missing_archs=$( for pkgnam in "${pkgname[@]}"; do archs=$( declare -f package_${pkgnam} | \ sed -n ' s/^\s*arch=(\(.\+\));$/\1/ T s/["'"'"']//g y/ /\n/ p ' | \ grep -xF 'any' || \ printf '%s\n' "${arch[@]}" ) for singleArch in ${archs}; do [ "${singleArch}" == "x86_64" ] || \ [ "${singleArch}" == "pentium4" ] || \ [ "${singleArch}" == "i686" ] || \ [ "${singleArch}" == "armv6h" ] || \ [ "${singleArch}" == "any" ] || \ continue pkgFile="${pkgnam}-${epoch:+${epoch}:}${pkgver}-${pkgrel}-${singleArch}.pkg.tar.xz" if ${force}; then printf '%s\n' "${singleArch}" else if [ "${repo}" == 'releng' ]; then [ ! -e "${pkgDir32}/${pkgFile}" ] && printf '%s\n' "${singleArch}" fi [ "${singleArch}" == "any" ] && \ for dir in ${pkgDir}/*; do [ ! -d "${dir}" ] && continue [ ! -e "${dir}/${pkgFile}" ] && printf '%s\n' "${singleArch}" done if [ ! -e "${pkgDir}/${singleArch}/${pkgFile}" ]; then printf '%s\n' "${singleArch}" fi fi done done | \ sort -u ) if [ ${#onlyArchs[@]} -ne 0 ]; then missing_archs=$( printf '%s\n' "${missing_archs}" | \ grep -xF "$(printf '%s\n' "${onlyArchs[@]}")" ) fi if [ ${#onlyPackages[@]} -ne 0 ]; then unset tmpOnlyArchs for ((i=0; i<${#onlyPackages[@]}; i++)); do [[ "${paket}" == "${onlyPackages[${i}]%:*}" ]] || continue [ -z "${onlyPackages[${i}]%%*:*}" ] || continue tmpOnlyArchs[${#tmpOnlyArchs[@]}]="${onlyPackages[${i}]##*:}" done if [ ${#tmpOnlyArchs[@]} -ne 0 ]; then missing_archs=$( printf '%s\n' "${missing_archs}" | \ grep -xF "$(printf '%s\n' "${tmpOnlyArchs[@]}")" ) fi fi if [ -n "${missing_archs}" ]; then for singleArch in ${missing_archs}; do if ! printf '%s\n' "${arch[@]}" | \ grep -qxF "${singleArch}"; then continue fi pakete[${#pakete[@]}]="${paket}" pakete_teile_und_archs[${#pakete_teile_und_archs[@]}]=$( for pn in "${pkgname[@]}"; do printf '%s:%s\n' \ "$( declare -f package_${pn} | \ sed -n ' s/^\s*arch=(\(.\+\));$/\1/ T s/["'"'"']//g y/ /\n/ p ' | \ grep -xF 'any' || \ printf '%s' "${singleArch}" )" \ "${pn}" done ) archs[${#archs[@]}]="${singleArch}" verss[${#verss[@]}]="${epoch:+${epoch}:}${pkgver}-${pkgrel}" if [ "${repo}" != 'archlinuxewe' ]; then repos[${#repos[@]}]="${repo} archlinuxewe" else repos[${#repos[@]}]="${repo}" fi done fi done # only print packages to be built? if ${printOnly}; then printf '%d scheduled builds:\n' "${#pakete[@]}" for ((i=0; i<${#pakete[@]}; i++)); do echo " ${pakete[${i}]} ${verss[${i}]} ${archs[${i}]}" printf ' %s\n' ${pakete_teile_und_archs[${i}]} done exit 0 fi maxErr=0 # generate current makepkg.conf, makepkg-i686.conf and makepkg-pentium4.conf cd "${tmpDir}" cp /etc/makepkg.conf . if ! patch -p0 -i makepkgSystem.patch; then cleanUp exit 1 fi sed 's|^\(BUILDENV=.*[^!]\)sign|\1!sign|' -i makepkg.conf eval "$( grep '^SRCDEST=' makepkg.conf )" if [ -n "${SRCDEST}" ]; then # update git sources find "${SRCDEST}" -mindepth 2 -maxdepth 2 -name 'HEAD' | \ sed 's|/[^/]*$||' | \ while read -r repo; do git -C "${repo}" fetch --all -p done # update svn sources find "${SRCDEST}" -mindepth 2 -maxdepth 2 -name '.svn' | \ sed 's|/[^/]*$||' | \ while read -r repo; do svn update "${repo}" done fi # build packages for ((i=0; i<${#pakete[@]}; i++)); do paket="${pakete[${i}]}" paket_teile_und_archs=(${pakete_teile_und_archs[${i}]}) arch="${archs[${i}]}" vers="${verss[${i}]}" repo="${repos[${i}]}" cd "${tmpDir}/${paket}" rm -rf --one-file-system src pkg # add PKGBUILD-patches to the PKGBUILD if grep -q '^\s*pkgbase=' PKGBUILD; then ( eval "$( sed -n '/^\s*pkgname=(/,/)/ p' PKGBUILD )" printf 'package_%s\n' "${pkgname[@]}" ) else printf 'package\n' fi \ | sponge \ | while read -r function; do sed -i ' /^'"${function}"'() {$/,/^}$/ { /^}$/ {'"$( ls -1 "${tmpDir}/"*".PKGBUILDpatch" | \ grep -vxF "$( sed ' s,^# skip \(\S\+\)\(\s\|$\).*$,'"${tmpDir}"'/\1.PKGBUILDpatch, t d ' PKGBUILD )" | \ sed 's/^/r /' )"' a } d } } ' PKGBUILD done case "${arch}" in 'any'|'i686'|'pentium4'|'x86_64') archlinuxewe-${arch/any/x86_64}-build ${cleanChroot} -- -- ${log} ${checkMakepkg} --holdver err[${i}]=$? if [ ${err[${i}]} -eq 0 ] && [ "${arch}" != 'x86_64' ] && [ "${arch}" != 'any' ]; then { printf '%s\n' \ "From: archlinux-reports@eckner.net" \ "To: buildmaster@archlinux32.org" \ "Subject: $(hostname) - report about installed ${arch} packages" \ "" find . \ -mindepth 1 \ -maxdepth 1 \ -name '*.pkg.tar.xz' \ -exec bsdtar -Oxf {} .BUILDINFO \; \ | sed ' s@^installed = \(.*\)$@/var/cache/archbuild/\1.pkg.tar.xz@ t d ' \ | xargs -r sha512sum \ 2>/dev/null \ | sed ' s@\s\+.*/@@ ' \ | xargs -r sha512sum \ | sed 's/^/stabilize: /' } | \ sendmailadvanced -t fi ;; 'armv6h') # armv6h is built on the raspberry pi SRCPKGDEST="${tmpDir}" makepkg -Sf err[${i}]=$? if [ ${err[${i}]} -eq 0 ]; then ssh eckner.net -p 15912 "$( printf 'set -e\n' printf 'sudo pacman -Sy >&2\n' printf 'rm -rf --one-file-system build\n' printf 'mkdir build\n' printf 'cd build\n' printf 'tar -xzf -\n' printf 'cd "%s"\n' "${paket}" printf 'makepkg %s --holdver --config ~/build/makepkg-armv6h.conf -fsr --noconfirm --asdeps >&2\n' \ "${log} ${checkMakepkg}" printf 'tar -czf - *.pkg.tar.xz' [ -n "${log}" ] && \ printf ' *.pkg.tar.xz-*.log' printf '\n' )" \ < "${tmpDir}/${paket}-${vers}.src.tar.gz" > "paket.tar.gz" err[${i}]=$? if [ ${err[${i}]} -eq 0 ]; then tar -xzf paket.tar.gz fi fi ;; *) printf 'unbekannte Architektur "%s"\n' "${arch}" err[${i}]=128 ;; esac for teil_und_arch in "${paket_teile_und_archs[@]}"; do if [ ${err[${i}]} -ne 0 ]; then break fi teil="${teil_und_arch#*:}" teil_arch="${teil_und_arch%%:*}" namcap "${tmpDir}/${paket}/${teil}-${vers}-${teil_arch}.pkg.tar.xz" > \ "${tmpDir}/namcap" err[${i}]=$? if [ -n "${log}" ]; then cat "${tmpDir}/namcap" > \ "${tmpDir}/${paket}/${teil}-namcap-${arch}.log" fi if [ ${err[${i}]} -ne 0 ]; then sed 's/^/namcap: /' "${tmpDir}/namcap" break fi if grep "$( { printf '%s\\|' \ "E: ELF file (.*) found in an ['\"]any['\"] package" if ! grep -qF " ${teil} ist absichtlich nicht any ohne ELF Dateien" PKGBUILD; then printf '%s\\|' \ "W: No ELF files and not an ['\"]any['\"] package" fi } | \ sed 's@\\|$@@' )" "${tmpDir}/namcap"; then err[${i}]=64 fi done rm -f "${tmpDir}/namcap" if ${abortOnMakepkgError} && [ ${err[${i}]} -ne 0 ]; then cleanUp exit ${err[${i}]} fi [ ${err[${i}]} -gt ${maxErr} ] && maxErr=${err[${i}]} eval "$( sed -n ' s/^GPGKEY=/key=/ T p ' "${tmpDir}/makepkg.conf" )" for teil_und_arch in "${paket_teile_und_archs[@]}"; do teil="${teil_und_arch#*:}" teil_arch="${teil_und_arch%%:*}" gpg --detach-sign -u "${key}" --no-armor ${teil}-${vers}-${teil_arch}.pkg.tar.xz done if ${upload} && [ ${err[${i}]} -eq 0 ]; then # upload package and update db cd "${tmpDir}/${paket}" printf '%s\n' "${paket_teile_und_archs[@]%%:*}" | \ sort -u | \ while read -r arch; do if [ "${paket}" = 'pacman-static' ] && \ [ "${arch}" = 'i686' ]; then tar -OxJf "${paket_teile_und_archs[0]#*:}" usr/bin/pacman-static 2>/dev/null > /srv/arch-mirror/sources.archlinux32/sources/pacman-static gpg --detach-sign -u "${key}" --yes --no-armor /srv/arch-mirror/sources.archlinux32/sources/pacman-static fi paket_teile=( $( printf '%s\n' "${paket_teile_und_archs[@]}" | \ sed -n ' s/^'"${arch}"':// T p ' ) ) tar -cf - $( printf " %s-${vers}-${arch}.pkg.tar.xz" "${paket_teile[@]}" printf " %s-${vers}-${arch}.pkg.tar.xz.sig" "${paket_teile[@]}" ) | \ ssh paule@127.0.0.1 "$( printf ' entferneAltePakete() { altePakete=$( ls -1 | \ grep "^$1-[^-]\+-[^-]\+-$2\.pkg\.tar\.xz\$" ) if [ -n "${altePakete}" ]; then if [ "$(basename "$(pwd)")" != "any" ]; then if [ "$(basename "$(pwd)")" == "releng" ]; then repo-remove releng.db.tar.gz "$1" else repo-remove %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1"' \ "${checkDbSig}" printf ' fi fi for altesPaket in ${altePakete}; do rm "${altesPaket}" if [ -e "${altesPaket}.sig" ] || [ -h "${altesPaket}.sig" ]; then rm "${altesPaket}.sig" fi done fi }' printf ' addPaket() { if [ "$(basename "$(pwd)")" == "releng" ]; then repo-add releng.db.tar.gz "$1" else repo-add %s -s -k 0x3CFB0AD8F60030F8 archlinuxewe.db.tar.gz "$1" fi }' \ "${checkDbSig}" for r in ${repo}; do if [ "${r}" = 'releng' ]; then printf ' cd "%s"' \ "${pkgDir32}" else printf ' cd "%s/%s"' \ "${pkgDir}" "${arch}" fi printf ' for teil in %s; do' \ "${paket_teile[*]}" printf ' entferneAltePakete "${teil}" "%s"' \ "${arch}" printf ' done' printf ' tar -xf - for teil in %s; do' \ "${paket_teile[*]}" if [ "${arch}" == "any" ] && [ "${r}" != 'releng' ]; then printf ' for lArch in $(ls ..); do [ ! -d "../${lArch}" ] && continue [ "${lArch}" == "any" ] && continue cd "../${lArch}" entferneAltePakete "${teil}" "any"' printf ' ln -s "../any/${teil}-%s-%s.pkg.tar.xz" "${teil}-%s-%s.pkg.tar.xz"' \ "${vers}" "${arch}" "${vers}" "${arch}" printf ' if [ -e "%s/any/${teil}-%s-%s.pkg.tar.xz.sig" ]' \ "${pkgDir}" "${vers}" "${arch}" printf '; then ln -s "../any/${teil}-%s-%s.pkg.tar.xz.sig" "${teil}-%s-%s.pkg.tar.xz.sig"' \ "${vers}" "${arch}" "${vers}" "${arch}" printf ' fi addPaket "${teil}-%s-%s.pkg.tar.xz"' \ "${vers}" "${arch}" printf ' done' else printf ' addPaket "${teil}-%s-%s.pkg.tar.xz"' \ "${vers}" "${arch}" fi printf ' done' done )" done fi done # check db if ${upload}; then ssh paule@127.0.0.1 "$( printf ' err=0 for arch in %s/* %s' \ "${pkgDir}" "${pkgDir32}" printf '; do [ ! -d "${arch}" ] && continue [ "${arch##*/}" == "any" ] && continue for paketFehler in $( \ ( if [ "$(basename "${arch}")" == 'releng' ]; then tar -Oxzf "${arch}/releng.db.tar.gz" else tar -Oxzf "${arch}/archlinuxewe.db.tar.gz" fi | \ grep -A1 "^%%FILENAME%%\$" | \ grep -v "^%%FILENAME%%\$" | \ grep -v -- "^--\$" ls "${arch}" | \ grep "^.*\.pkg\.tar\.xz\$" ) | \ sort | \ uniq -u ); do err=1 if [ -e "${arch}/${paketFehler}" ]; then >&2 echo "FEHLER: Paket ${arch}/${paketFehler} ist nur im Verzeichnisbaum vorhanden." else >&2 echo "FEHLER: Paket ${arch}/${paketFehler} ist nur in der Datenbank vorhanden." fi done for signaturWarnungen in $( \ ls "${arch}" | \ grep "^.*\.pkg\.tar\.xz\(\.sig\)\?\$" | \ sed "s|\.sig\$||" | \ sort | \ uniq -u ); do if [ -e "${arch}/${signaturWarnungen}" ]; then >&2 echo "WARNUNG: Paket ${arch}/${signaturWarnungen} hat keine Signatur." else err=1 >&2 echo "FEHLER: Zur Signatur ${arch}/${signaturWarnungen}.sig gibt es kein Paket." fi done done exit ${err}' )" fi cleanUp exit ${maxErr} else if ${checkSanity}; then # check packages err=false namcap */PKGBUILD || err=true for paket in *; do [ -r "${paket}/PKGBUILD" ] || continue if ! grep -q "^# Maintainer\s*:\s\+Erich Eckner\s\+\$" "${paket}/PKGBUILD" || \ grep "^# Maintainer\s*:" "${paket}/PKGBUILD" | \ grep -vq "^# Maintainer\s*:\s\+Erich Eckner\s\+\$"; then >&2 echo "${paket}/PKGBUILD hat falsche(n) Maintainer" if [ -d "${paket}/.git" ]; then continue fi err=true fi done if ${err}; then exit 1 fi fi cd "${pkgSrcDir}" git push err=$? if [ ${err} -ne 0 ]; then exit ${err} fi ssh -p 22222 makepkg@141.35.51.219 "~/archPackagesUpdate" "${args[@]}" err=$? if [ ${err} -ne 0 ]; then exit ${err} fi fi