bin/build-packages bin/common-functions: use git archive thoroughly instead of git checkout

2 files changed, 79 insertions, 122 deletions

diff --git a/bin/build-packages b/bin/build-packages
index 85be27f..1c0f1c9 100755
--- a/bin/build-packages
+++ b/bin/build-packages
@@ -135,24 +135,37 @@ while [ ${count} -ne 0 ]; do
 
       for repo_name in ${repo_names}; do
         eval repo_path='$repo_paths__'"${repo_name}"
-        git -C "${repo_path}" clean -df
-        git -C "${repo_path}" reset --hard
-        git -C "${repo_path}" checkout master
-        git -C "${repo_path}" pull || true
+        git -C "${repo_path}" fetch
       done
 
-      eval git -C "$(printf '$repo_paths__%s' "$(find_repository_with_commit "${git_revision}")")" checkout "${git_revision}" > /dev/null 2>&1
-      git -C "${repo_paths__archlinux32}" checkout "${mod_git_revision}" > /dev/null 2>&1
+      git_repo="$(find_repository_with_commit "${git_revision}")"
 
-      PKGBUILD="$(find_pkgbuild "${package}" "${repository}")"
+      find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
 
-      if [ ! -r "${PKGBUILD}" ]; then
-        echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'"
-        exit 1
+      bail_out() {
+        err=$?
+        if [ -n $1 ]; then
+          err=$1
+        fi
+        cd "${base_dir}"
+        rm -rf --one-file-system "${tmp_dir}"
+        exit ${err}
+      }
+      tmp_dir="$(mktemp -d "${work_dir}/tmp.XXXXXX")"
+
+      if [ -n "${PKGBUILD}" ]; then
+        eval git -C "$(printf '$repo_paths__%s' "${git_repo}")/${PKGBUILD%/*}" archive "${git_revision}" | \
+          tar -x -C "${tmp_dir}"
+      fi
+      if [ -n "${PKGBUILD_mod}" ]; then
+        git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" | \
+          tar -x --overwrite --exclude 'PKGBUILD' -C "${tmp_dir}"
+        git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" | \
+          tar -Ox 'PKGBUILD' >> \
+          "${tmp_dir}/PKGBUILD"
       fi
 
-      cd "${PKGBUILD%/*}"
-      apply_package_customizations
+      cd "${tmp_dir}"
       success=false
       for mirrored_source in false true; do
 
@@ -180,7 +193,6 @@ while [ ${count} -ne 0 ]; do
                 ta
               '
           )"
-          git_repo="$(find_git_repository_to_package_repository "${repository}")"
           if ! wget -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
             # we can't improve anything
             break
@@ -191,19 +203,25 @@ while [ ${count} -ne 0 ]; do
         for parameters in '' '-c'; do
           rm -f *".pkg.tar.xz" *".pkg.tar.xz.sig"
 
-          if staging-i686-build ${parameters} -- archlinux32-keyring > \
+          >&2 printf 'building package "%s" ...' "${package}"
+          # by piping the log, we don't see anything in the terminal,
+          # but all ways to duplicate the logs seem pretty elaborate
+          if staging-i686-build ${parameters} > \
             "$(
               date -u --iso-8601=seconds | \
                 cut -d+ -f1
             ).build-log" 2>&1; then
             # build successful
+            >&2 printf ' ok.\n'
             ls -1 *.pkg.tar.xz | \
-              xargs -rn1 gpg --local-user="${package_key}" --detach-sign
+              xargs -rn1 gpg --local-user="${package_key}" --detach-sign || \
+              bail_out
             tar -cf 'package.tar' $(
                 ls -1 | \
                   grep '\.pkg\.tar\.xz\(\.sig\)\?$' | \
                   grep -v -- '-debug-'
-              )
+              ) || \
+              bail_out
             while true; do
               err=0
               ssh \
@@ -228,20 +246,21 @@ while [ ${count} -ne 0 ]; do
                 ;;
                 3)
                   >&2 echo "'return-assignment' reports a signature error."
-                  exit 1
+                  bail_out 1
                 ;;
                 4)
                   >&2 echo "'return-assignment' reports too many or missing packages."
-                  exit 1
+                  bail_out 1
                 ;;
                 *)
                   >&2 echo "unknown return code ${err} from 'return-assignment'"
-                  exit 1
+                  bail_out 1
               esac
             done
             success=true
             break
           fi
+          >&2 printf ' failed.\n'
         done
         if ${success}; then
           break
@@ -250,10 +269,14 @@ while [ ${count} -ne 0 ]; do
 
       if ! ${success}; then
         for log in *'.build-log'; do
-          gzip "${log}"
+          if [ -f "${log}" ]; then
+            gzip "${log}" || \
+              bail_out
+          fi
         done
         tar -cf 'build-logs.gz.tar' \
-          *'.build-log.gz'
+          *'.build-log.gz' || \
+          continue # this might happen if we didn't create any logs
         while ! ssh \
           -i "${master_build_server_identity}" \
           -p "${master_build_server_port}" \
@@ -264,10 +287,14 @@ while [ ${count} -ne 0 ]; do
         done
         if ${exit_after_failure}; then
           >&2 echo 'Build failed, exiting now'
-          exit
+          bail_out 0
         fi
       fi
 
+      # clean up tmp_dir
+      cd "${base_dir}"
+      rm -rf --one-file-system "${tmp_dir}"
+
       continue
 
     ;;
diff --git a/bin/common-functions b/bin/common-functions
index 59b7577..442f5b2 100755
--- a/bin/common-functions
+++ b/bin/common-functions
@@ -6,91 +6,44 @@
 
 #  mangle $arch in PKBUILDs to contain i486, i586, i686
 
-# find_pkgbuild package repository
-# find the PKGBUILD of $package from $repository
+# find_pkgbuilds package repository git_repository git_revision mod_git_revision
+# find the PKGBUILD and modification of $package from $repository
+# sets $PKGBUILD and $PKGBUILD_mod
 
-find_pkgbuild() {
+find_pkgbuilds() {
 
   local package="$1"
   local repository="$2"
+  local git_repository="$3"
+  local git_revision="$4"
+  local mod_git_revision="$5"
 
-  local PKGBUILD=''
-  local repo
-  local file
-  local package_path
-
-  if [ -f "${repo_paths__archlinux32}/${repository}/${package}/PKGBUILD" ]; then
-    # If this package has some modification,
-    repo="$(find_git_repository_to_package_repository "${repository}")"
-    eval package_path="$(printf '$repo_paths__%s' "${repo}")/${package}"
-    if ! [ -d "${package_path}" ]; then
-      # create some dummy files if it is also new.
-      mkdir -p "${package_path}/repos/${repository}-x86_64"
-      touch "${package_path}/repos/${repository}-x86_64/PKGBUILD"
-    fi
-  fi
-
-  for repo in ${repo_names}; do
-    if [ "${repo}" = "archlinux32" ]; then
-      # this is not a repository of packages
-      continue
-    fi
-    eval package_path="$(printf '$repo_paths__%s' "${repo}")/${package}"
-    if ! [ -d "${package_path}" ]; then
-      continue
-    fi
-    PKGBUILD="$(
-      ls "${package_path}/repos/${repository}-"*"/PKGBUILD" 2> /dev/null | \
-        tr ' ' '\n' | \
-        grep -v -- '-i686/PKGBUILD$' | \
-        grep -v -- '[-/]\(staging\|testing\)-[^/]\+/PKGBUILD$' | \
-        sort | \
-        tail -n1
-    )"
-    if [ -n "${PKGBUILD}" ]; then
-      echo "${PKGBUILD}"
-      break
-    fi
-  done
-}
+  PKGBUILD="$(
+    eval git -C "$(printf '$repo_paths__%s' "${git_repository}")" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
+      tar -t 2> /dev/null | \
+      grep "^$(str_to_regex "${package}/repos/${repository}")"'-.*/PKGBUILD$' | \
+      grep -v -- '-i686/PKGBUILD$' | \
+      grep -v -- '[-/]\(staging\|testing\)-[^/]\+/PKGBUILD$' | \
+      sort | \
+      tail -n1
+  )"
 
-# apply customizations to a package
-#  (to be executed in the package's directory)
+  PKGBUILD_mod="$(
+    git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
+      tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
+  )" || true
 
-apply_package_customizations() {
-  if [ ! -f 'PKGBUILD' ]; then
-    >&2 echo 'PKGBUILD not found.'
-    pwd
-    exit 1
+  if [ -z "${PKGBUILD}" ] && \
+    [ -z "${PKGBUILD_mod}" ]; then
+    >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
+      "${package}" \
+      "${repository}" \
+      "${git_repository}" \
+      "${git_revision}" \
+      "${mod_git_revision}"
+    return 1
   fi
 
-  local repo
-  local package
-
-  repo="$(pwd)"
-  package="${repo%/*/*}"
-  package="${package##*/}"
-  repo="${repo##*/}"
-  repo="${repo%-any}"
-  repo="${repo%-x86_64}"
-
-  if [ ! -f 'PKGBUILD.changes-applied' ]; then
-    # add i686 to the arch list
-    sed '/^arch=[^#]*any/!s|^\(arch=(\)\([^#]*)\)\s*\(#.*\)\?$|\1'"'i686'"' \2|' -i 'PKGBUILD'
-    if [ -f "${repo_paths__archlinux32}/${repo}/${package}/PKGBUILD" ]; then
-      # If this package has modifications (or is new), apply them now:
-      # append PKGBUILD
-      cat "${repo_paths__archlinux32}/${repo}/${package}/PKGBUILD" >> \
-        'PKGBUILD'
-      # copy (and overwrite) other files
-      for file in "${repo_paths__archlinux32}/${repo}/${package}/"*; do
-        if [ -f "${file}" ] && [ "${file##*/}" != 'PKGBUILD' ]; then
-          cp "${file}" ./
-        fi
-      done
-    fi
-    touch 'PKGBUILD.changes-applied'
-  fi
 }
 
 # find_repository_with_commit commit
@@ -382,30 +335,7 @@ make_source_info() {
     return 1
   fi
 
-  PKGBUILD="$(
-    eval git -C "$(printf '$repo_paths__%s' "${git_repo}")" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
-      tar -t 2> /dev/null | \
-      grep "^$(str_to_regex "${package}/repos/${repository}")"'-.*/PKGBUILD$' | \
-      grep -v -- '-i686/PKGBUILD$' | \
-      grep -v -- '[-/]\(staging\|testing\)-[^/]\+/PKGBUILD$' | \
-      sort | \
-      tail -n1
-  )"
-
-  PKGBUILD_mod="$(
-    git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
-      tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
-  )" || true
-
-  if [ -z "${PKGBUILD}" ] && \
-    [ -z "${PKGBUILD_mod}" ]; then
-    >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package %s from %s, revisions %s and %s.\n' \
-      "${package}" \
-      "${repository}" \
-      "${git_revision}" \
-      "${mod_git_revision}"
-    return 1
-  fi
+  find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
 
   if [ -n "${PKGBUILD}" ]; then
     content="$(