bin/harvest-commit-times: harvest 10 commit times at a time and then update the database

1 files changed, 53 insertions, 46 deletions

diff --git a/bin/harvest-commit-times b/bin/harvest-commit-times
index 1f8c2e0..38d84c4 100755
--- a/bin/harvest-commit-times
+++ b/bin/harvest-commit-times
@@ -61,53 +61,60 @@ fi
 tmp_file=$(mktemp 'tmp.harvest-commit-times.XXXXXXXXXX' --tmpdir)
 trap 'rm "${tmp_file}"' EXIT
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT'
-  printf ' `package_sources`.`id`,'
-  printf '`package_sources`.`pkgbase`,'
-  printf '`package_sources`.`git_revision`,'
-  printf '`package_sources`.`mod_git_revision`,'
-  printf '`upstream_repositories`.`name`,'
-  printf '`git_repositories`.`name`'
-  printf ' FROM `package_sources`'
-  mysql_join_package_sources_upstream_repositories
-  mysql_join_upstream_repositories_git_repositories
-  printf ' WHERE `package_sources`.`commit_time`="0000-00-00 00:00:00"'
-  printf ' AND `package_sources`.`uses_upstream`'
-} | \
-  mysql_run_query 'unimportant' | \
-  sponge | \
-  while read -r id pkgbase git_revision mod_git_revision repository git_repository; do
-    eval 'git_directory="${repo_paths__'"${git_repository}"'}"'
-    find_pkgbuilds "${pkgbase}" "${repository}" "${git_repository}" "${git_revision}" "${mod_git_revision}"
-    commit_time=$(
-      # shellcheck disable=SC2154
-      git -C "${git_directory}" log -n 1 --pretty=format:%ct "${git_revision}" -- "${PKGBUILD}"
-    )
-    if [ -n "${commit_time}" ]; then
-      printf '%s\t%s\n' \
-        "${id}" \
-        "${commit_time}"
-    else
-      >&2 printf 'Package source %s is not available in git.\n' \
-        "${id}"
-      exit 2
-    fi
-  done > \
-  "${tmp_file}"
+while true; do
 
-if [ -s "${tmp_file}" ]; then
   # shellcheck disable=SC2016
   {
-    printf 'CREATE TEMPORARY TABLE `cts` (`id` BIGINT, `ct` BIGINT);\n'
-    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `cts`(`id`,`ct`);\n' \
-      "${tmp_file}"
-    printf 'UPDATE `cts`'
-    printf ' JOIN `package_sources`'
-    printf ' ON `package_sources`.`id`=`cts`.`id`'
-    printf ' AND `package_sources`.`commit_time`="0000-00-00 00:00:00"'
-    printf ' SET `package_sources`.`commit_time`=from_unixtime(`cts`.`ct`);\n'
+    printf 'SELECT'
+    printf ' `package_sources`.`id`,'
+    printf '`package_sources`.`pkgbase`,'
+    printf '`package_sources`.`git_revision`,'
+    printf '`package_sources`.`mod_git_revision`,'
+    printf '`upstream_repositories`.`name`,'
+    printf '`git_repositories`.`name`'
+    printf ' FROM `package_sources`'
+    mysql_join_package_sources_upstream_repositories
+    mysql_join_upstream_repositories_git_repositories
+    printf ' WHERE `package_sources`.`commit_time`="0000-00-00 00:00:00"'
+    printf ' AND `package_sources`.`uses_upstream`'
+    printf ' LIMIT 10'
   } | \
-    mysql_run_query 'unimportant'
-fi
+    mysql_run_query 'unimportant' | \
+    sponge | \
+    while read -r id pkgbase git_revision mod_git_revision repository git_repository; do
+      eval 'git_directory="${repo_paths__'"${git_repository}"'}"'
+      find_pkgbuilds "${pkgbase}" "${repository}" "${git_repository}" "${git_revision}" "${mod_git_revision}"
+      commit_time=$(
+        # shellcheck disable=SC2154
+        git -C "${git_directory}" log -n 1 --pretty=format:%ct "${git_revision}" -- "${PKGBUILD}"
+      )
+      if [ -n "${commit_time}" ]; then
+        printf '%s\t%s\n' \
+          "${id}" \
+          "${commit_time}"
+      else
+        >&2 printf 'Package source %s is not available in git.\n' \
+          "${id}"
+        exit 2
+      fi
+    done > \
+    "${tmp_file}"
+
+  if [ -s "${tmp_file}" ]; then
+    # shellcheck disable=SC2016
+    {
+      printf 'CREATE TEMPORARY TABLE `cts` (`id` BIGINT, `ct` BIGINT);\n'
+      printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `cts`(`id`,`ct`);\n' \
+        "${tmp_file}"
+      printf 'UPDATE `cts`'
+      printf ' JOIN `package_sources`'
+      printf ' ON `package_sources`.`id`=`cts`.`id`'
+      printf ' AND `package_sources`.`commit_time`="0000-00-00 00:00:00"'
+      printf ' SET `package_sources`.`commit_time`=from_unixtime(`cts`.`ct`);\n'
+    } | \
+      mysql_run_query 'unimportant'
+  else
+    break
+  fi
+
+done