bin/get-package-updates: use 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`' instead of 'INSERT IGNORE INTO `%s` VALUES ...'

1 files changed, 76 insertions, 100 deletions

diff --git a/bin/get-package-updates b/bin/get-package-updates
index 1b3bd68..724e261 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -207,7 +207,13 @@ if ! verbose_flock -s ${block_flag} 8; then
   exit
 fi
 
-trap mysql_cleanup EXIT
+cleanup() {
+  mysql_cleanup
+  rm -rf --one-file-system "${tmp_dir:?}"
+}
+tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir)
+
+trap cleanup EXIT
 
 # shellcheck disable=SC2119
 mysql_cleanup
@@ -422,107 +428,81 @@ echo 'apply blacklisting'
 # ignore blacklisted packages and dependent packages
 # this is the first time when all the information is available and up to date
 
-# TODO: use 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`' instead of
-# "INSERT IGNORE INTO `%s` VALUES ..."
-
-black_listed=''
-black_listed_new=$(
+touch "${tmp_dir}/black-listed"
+{
+  git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+    tar -Ox 'blacklist' | \
+    sed '
+      s/\s*#.*$//
+      /^\s*$/d
+    '
+  if [ -n "${test_exclusion}" ]; then
+    echo "${test_exclusion}"
+  fi
+  # shellcheck disable=SC2016
   {
-    {
-      git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
-        tar -Ox 'blacklist' | \
-        sed '
-          s/\s*#.*$//
-          /^\s*$/d
-        '
-      if [ -n "${test_exclusion}" ]; then
-        echo "${test_exclusion}"
-      fi
-    } | \
-      base64_encode_each
-    # shellcheck disable=SC2016
-    {
-      printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")'
-      printf ' FROM `package_sources`'
-      printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
-    } | \
-      mysql_run_query
+    printf 'SELECT DISTINCT `package_sources`.`pkgbase`'
+    printf ' FROM `package_sources`'
+    printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
   } | \
-    sort -u
-)
+    mysql_run_query
+} | \
+  sort -u > \
+  "${tmp_dir}/black-listed-new"
 
-while [ -n "${black_listed_new}" ]; do
-  black_listed=$(
-    printf '%s\n' "${black_listed}" "${black_listed_new}" | \
-      grep -vxF '' | \
-      sort -u
-  )
-  black_listed_new=$(
-    {
-      printf '%s\n' "${black_listed}" "${black_listed}"
+while [ -s "${tmp_dir}/black-listed-new" ]; do
+  cat \
+    "${tmp_dir}/black-listed-new" \
+    "${tmp_dir}/black-listed" | \
+    sort -u | \
+    sponge "${tmp_dir}/black-listed"
+  {
+    sed 'p' "${tmp_dir}/black-listed"
 
-      # shellcheck disable=SC2016
-      {
-        printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
-        printf 'INSERT INTO `bl` (`pkgbase`) VALUES '
-        printf '%s\n' "${black_listed}" | \
-          sort -u | \
-          sed '
-            s/^/(from_base64("/
-            s/$/")),/
-            $ s/,$/;/
-          '
-        printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")'
-        printf ' FROM `package_sources` AS `a_ps`'
-        mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
-        mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
-        mysql_join_binary_packages_dependencies 'a_bp'
-        mysql_join_dependencies_dependency_types
-        printf ' AND (`dependency_types`.`relevant_for_building`'
-        printf ' OR `dependency_types`.`relevant_for_binary_packages`)'
+    # shellcheck disable=SC2016
+    {
+      printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n'
+      printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `bl` (`pkgbase`);\n' \
+        "${tmp_dir}/black-listed"
+      printf 'SELECT `a_ps`.`pkgbase`'
+      printf ' FROM `package_sources` AS `a_ps`'
+      mysql_join_package_sources_build_assignments 'a_ps' 'a_ba'
+      mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp'
+      mysql_join_binary_packages_dependencies 'a_bp'
+      mysql_join_dependencies_dependency_types
+      printf ' AND (`dependency_types`.`relevant_for_building`'
+      printf ' OR `dependency_types`.`relevant_for_binary_packages`)'
+      printf ' WHERE NOT EXISTS ('
+        printf ' SELECT 1'
+        printf ' FROM `install_target_providers`'
         printf ' WHERE NOT EXISTS ('
           printf ' SELECT 1'
-          printf ' FROM `install_target_providers`'
-          printf ' WHERE NOT EXISTS ('
-            printf ' SELECT 1'
-            printf ' FROM `bl`'
-            printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
-            mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
-            mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
-            printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
-          printf ')'
-          printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
-        printf ') AND EXISTS ('
-        # TODO: This should be corrected at the root: automatic install targets, which are bogus should
-        #       not be added in the first place - but how do we detect that? -> add exceptions!
-          printf ' SELECT 1'
-          printf ' FROM `install_target_providers`'
-          printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
-        printf ');\n'
-      } | \
-        mysql_run_query | \
-        sort -u
+          printf ' FROM `bl`'
+          printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`'
+          mysql_join_package_sources_build_assignments 'b_ps' 'b_ba'
+          mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp'
+          printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`'
+        printf ')'
+        printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+      printf ') AND EXISTS ('
+      # TODO: This should be corrected at the root: automatic install targets, which are bogus should
+      #       not be added in the first place - but how do we detect that? -> add exceptions!
+        printf ' SELECT 1'
+        printf ' FROM `install_target_providers`'
+        printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+      printf ');\n'
     } | \
-      grep -vxF '' | \
-      sort | \
-      uniq -u
-  )
+      mysql_run_query | \
+      sort -u
+  } | \
+    sort | \
+    uniq -u | \
+    sponge "${tmp_dir}/black-listed-new"
 done
 
-black_listed=$(
-  printf '%s\n' "${black_listed}" | \
-  while read -r line; do
-    printf '%s' \
-      "${line}" | \
-      base64 -d
-    printf '\n'
-  done
-)
-
 deletion_list_count=$(
-  # shellcheck disable=SC2086
-  printf '%s\n' ${black_listed} | \
-    wc -l
+  wc -l < \
+    "${tmp_dir}/black-listed"
 )
 if [ "${deletion_list_count}" -gt 1000 ]; then
   >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \
@@ -550,14 +530,10 @@ if [ "${deletion_list_count}" -gt 1000 ]; then
 fi
 rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
 
-echo "${black_listed}" | \
-  while read -r package; do
-    if [ -z "${package}" ]; then
-      continue
-    fi
-    delete_package "${package}"
-  done
-
+while read -r package; do
+  delete_package "${package}"
+done < \
+  "${tmp_dir}/black-listed"
 
 if [ -n "${test_exclusion}" ]; then
   # TODO: reimplement test_exclusion with information from the database