lib/mysql-functions: solve a few todos in mysql_generate_package_metadata() - we now have versioned install targets!

1 files changed, 216 insertions, 308 deletions

diff --git a/lib/mysql-functions b/lib/mysql-functions
index 6c8154c..40843e3 100755
--- a/lib/mysql-functions
+++ b/lib/mysql-functions
@@ -210,6 +210,8 @@ mysql_add_package_source() {
 
 mysql_generate_package_metadata() {
 
+  mysql_load_min_and_max_versions
+
   ( # new shell is intentional
     current_repository_id="$1"
     package="$2"
@@ -260,9 +262,6 @@ mysql_generate_package_metadata() {
       mysql_add_package_source "${pkgbase}" "${git_revision}" "${old_git_revision}" "${mod_git_revision}" "${repository}" "${temp_dir}/SRCINFO"
     )
 
-    # TODO: architectures should be identified by id, not "(SELECT ...)"
-
-    # add the build assignment(s)
     {
       sed -n '
         s/^\tarch = //
@@ -272,342 +271,251 @@ mysql_generate_package_metadata() {
         sort -u | \
         grep -vxF 'any' || \
         echo 'any'
-    } | \
-      base64_encode_each | \
-      while read -r arch; do
-        printf 'INSERT INTO `build_assignments` (`package_source`,`architecture`,`is_blocked`,`is_broken`,`priority`)'
-        printf ' SELECT %s,`architectures`.`id`,NULL,0,0' \
-          "${package_source_id}"
-        printf ' FROM `architectures`'
-        printf ' WHERE `architectures`.`name` = from_base64("%s")' \
-          "${arch}"
-        printf ' ON DUPLICATE KEY UPDATE `id`=LAST_INSERT_ID(`build_assignments`.`id`);\n'
-        printf 'SELECT "%s",LAST_INSERT_ID();\n' \
-          "${arch}"
-      done | \
-      mysql_run_query > \
-      "${temp_dir}/build-assignments.id"
+    } > \
+      "${temp_dir}/architectures"
 
     # select any specific arch (which will be building the 'any' part of a split package)
     any_arch=$(
-      cut -f1 < \
-        "${temp_dir}/build-assignments.id" | \
-        sort | \
-        head -n1 | \
-        tr -d '\n' | \
-        base64 -w0
+      tail -n1 "${temp_dir}/architectures"
     )
 
     # iterate over all pkgnames
     grep '^pkgname = ' "${temp_dir}/SRCINFO" | \
       cut -d' ' -f3 | \
       while read -r pkgname; do
-        pkgname64=$(
-          printf '%s' "${pkgname}" | \
-            base64 -w0
-        )
         sed -n '
           /^pkgbase = \|^pkgname = '"$(str_to_regex "${pkgname}")"'$/,/^$/ {
+            s/^pkgname/\t\0/
             /^\S/d
+            /^\s*$/d
             s/^\s*//
             p
           }
         ' "${temp_dir}/SRCINFO" > \
-          "${temp_dir}/BINARYINFO.${pkgname64}"
+        "${temp_dir}/SRCINFO.tmp"
 
-        # iterate over all archs
-        grep '^arch = ' "${temp_dir}/BINARYINFO.${pkgname64}" | \
-          cut -d' ' -f3 | \
-          while read -r arch; do
-            arch64=$(
-              printf '%s' "${arch}" | \
-                base64 -w0
-            )
-            {
-              {
-                # this binary package is either built by the build_assignment
-                # with the identical arch (if existent) or by the one
-                # with arch=$any_arch
-                grep "^${arch64}"'\s' "${temp_dir}/build-assignments.id" || \
-                  grep "^${any_arch}"'\s' "${temp_dir}/build-assignments.id"
-              } | \
-                sed '
-                  s/^.*\s/build-assignment-id = /
-                '
-              sed '
-                s/^\(\S\+\)_'"${arch}"' = /\1 = /
-              ' "${temp_dir}/BINARYINFO.${pkgname64}"
-            } > \
-              "${temp_dir}/ARCHINFO ${pkgname64} ${arch64}"
-          done
-      done
-
-    # TODO: Data should be loaded by "LOAD DATA LOCAL INFILE ...". This
-    # has three advantages: 1st, we separate data from the commands
-    # (aesthetic), 2nd, we can prepare/check the data to load "offline",
-    # 3rd, we are less sensitive to errors in the middle of the commit.
-    # For this we need some intermediate files which are loaded into
-    # temporary tables:
-    #   - pkgbase -> build_assignment (multiple archs possible!)
-    #   - build_assignment -> pkgname (split packages possible!)
-    #   - pkgname -> dependency
-    #   - pkgname -> provides
-
-    # now each "ARCHINFO $pkgname $arch" file represents one binary package
-    find "${temp_dir}" -mindepth 1 -maxdepth 1 -name 'ARCHINFO * *' -printf '%f\n' | \
-      while read -r _ pkgname arch; do
-        build_assignment_id=$(
-          grep '^build-assignment-id = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3
-        )
-        pkgver=$(
-          grep '^pkgver = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            tr -d '\n' | \
-            base64 -w0
-        )
-        pkgrel=$(
-          grep '^pkgrel = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/\.[0-9]\+$//' | \
-            tr -d '\n' | \
-            base64 -w0
-        )
         epoch=$(
           {
-            grep '^epoch = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" || \
+            grep -m1 '^epoch = ' "${temp_dir}/SRCINFO.tmp" || \
               echo 'epoch = 0'
           } | \
-            cut -d' ' -f3 | \
-            tr -d '\n' | \
-            base64 -w0
-        )
-        # TODO: allow versioned install_targets
-        provides=$(
-          grep '^\(groups\|provides\) = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        makedepends=$(
-          grep '^makedepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
-        )
-        checkdepends=$(
-          grep '^checkdepends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
+            cut -d' ' -f3
         )
-        rundepends=$(
-          grep '^depends = ' "${temp_dir}/ARCHINFO ${pkgname} ${arch}" | \
-            cut -d' ' -f3 | \
-            sed 's/[<>=].*$//' | \
-            base64_encode_each
+        pkgver=$(
+          grep -m1 '^pkgver = ' "${temp_dir}/SRCINFO.tmp" | \
+            cut -d' ' -f3
         )
-        sub_pkgrel=$(
-          printf '(SELECT COALESCE('
-          # do not add binary packages which are currently on the
-          # build-list or in $current_repository_id (beware of split
-          # packages!)
-          printf '(SELECT `sub_pkgrel` FROM `binary_packages`'
-          mysql_join_binary_packages_architectures
-          mysql_join_binary_packages_binary_packages_in_repositories
-          printf ' WHERE'
-          printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-            'epoch' "${epoch}" \
-            'pkgver' "${pkgver}" \
-            'pkgrel' "${pkgrel}" \
-            'pkgname' "${pkgname}"
-          printf ' `architectures`.`name`=from_base64("%s")' \
-            "${arch}"
-          printf ' AND `binary_packages_in_repositories`.`repository` IN (%s,%s)),' \
-            "${repository_ids__any_build_list}" \
-            "${current_repository_id}"
-          # max(sub_pkgrel)+1
-          printf '(SELECT 1+MAX(`binary_packages`.`sub_pkgrel`) FROM `binary_packages`'
-          mysql_join_binary_packages_architectures
-          printf ' WHERE'
-          printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-            'epoch' "${epoch}" \
-            'pkgver' "${pkgver}" \
-            'pkgrel' "${pkgrel}" \
-            'pkgname' "${pkgname}"
-          if printf '%s' "${arch}" | base64 -d | grep -qxF 'any'; then
-            # 'any' gets higher sub_pkgrel than any architecture
-            printf ' 1'
-          else
-            # not-'any' gets higher sub_pkgrel than same or 'any' architecture
-            printf ' (`architectures`.`name`=from_base64("%s") OR `architectures`.`name`="any")' \
-              "${arch}"
-          fi
-          printf ')'
-          printf ',0))'
+        pkgrel=$(
+          grep -m1 '^pkgrel = ' "${temp_dir}/SRCINFO.tmp" | \
+            cut -d' ' -f3
         )
-        { # TODO: identify arch by id
-          printf 'INSERT IGNORE INTO `binary_packages` ('
-          printf '`%s`,' \
-            'build_assignment' \
-            'architecture' \
-            'epoch' \
-            'pkgver' \
-            'pkgrel' \
-            'pkgname' \
-            'sub_pkgrel' \
-            'has_issues' \
-            'is_tested'
-          printf ') SELECT '
-          printf '%s,' \
-            "${build_assignment_id}"
-          printf '`%s`.`id`,' \
-            'architectures'
-          printf 'from_base64("%s"),' \
-            "${epoch}" \
-            "${pkgver}" \
-            "${pkgrel}" \
-            "${pkgname}"
-          printf '%s,0,0 FROM `architectures`' \
-            "${sub_pkgrel}"
-          printf ' WHERE'
-          printf ' `architectures`.`name` = from_base64("%s")' \
-            "${arch}"
-          printf ';\n'
-          printf 'INSERT IGNORE INTO `binary_packages_in_repositories` (`package`,`repository`,`is_to_be_deleted`)'
-          printf ' SELECT LAST_INSERT_ID(),%s,0;\n' \
-            "${current_repository_id}"
-          printf 'COMMIT;\n'
-        } | \
-          sed '
-            s|,)|)|g
-            s| JOIN WHERE | WHERE |
-            s| AND;$|;|
-          ' >> \
-          "${temp_dir}/add-binary-packages-command"
-        {
-          printf 'CREATE TEMPORARY TABLE `%s` (`name` VARCHAR(64));\n' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends'
-
-          printf 'INSERT INTO `provides` VALUES\n'
-          echo "${provides}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-            '
-          printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"),"-",from_base64("%s"))),\n' \
-            "${pkgname}" \
-            "${epoch}" \
-            "${pkgver}" \
-            "${pkgrel}"
-          printf ' (CONCAT(from_base64("%s"),"-",from_base64("%s"),":",from_base64("%s"))),\n' \
-            "${pkgname}" \
-            "${epoch}" \
-            "${pkgver}"
-          printf ' (from_base64("%s"));\n' \
-            "${pkgname}"
-
-          printf 'INSERT INTO `rundepends` VALUES\n'
-          echo "${rundepends}" | \
-            sort -u | \
-            grep -vxF '' | \
-            sed '
-              s|^| (from_base64("|
-              s|$|")),|
-            '
-          printf ' ("base");\n'
 
-          echo "${checkdepends}" | \
-            sort -u | \
-            grep -vxF '' | \
+        # iterate over all archs
+        grep '^arch = ' "${temp_dir}/SRCINFO.tmp" | \
+          cut -d' ' -f3 | \
+          while read -r arch; do
+            build_arch=$(
+              # this binary package is either built by the build_assignment
+              # with the identical arch (if existent) or by the one
+              # with arch=$any_arch
+              grep -xF "${arch}" "${temp_dir}/architectures" || \
+                echo "${any_arch}"
+            )
             sed '
-              s|^| (from_base64("|
-              s|$|")),|
-              1 s/^/INSERT INTO `checkdepends` VALUES \n/
-              $ s/,$/;/
-            '
-
-          printf 'INSERT INTO `makedepends` VALUES\n'
-          echo "${makedepends}" | \
-            sort -u | \
-            grep -vxF '' | \
+              s/^\(\S\+\)_'"${arch}"' = /\1 = /
+              s/^pkgname = \(\S\+\)$/\0\nprovides = \1='"${epoch}"':'"${pkgver}"'\ndepends = base\nmakedepends = base-devel/
+            ' "${temp_dir}/SRCINFO.tmp" | \
+            sed '
+              s/^arch = \S\+$/arch/
+              t
+              s/^\(groups\|provides\) = /provides\t/
+              t mangle_version
+              s/^makedepends = /makedepends\t/
+              t mangle_version
+              s/^checkdepends = /checkdepends\t/
+              t mangle_version
+              s/^depends = /rundepends\t/
+              t mangle_version
+              d
+              :mangle_version
+              s/\(=\|<\|<=\|>=\|>\)\([^[:space:]-]\+\)$/\t\1\t\2/
+              t split_epoch
+              h
+              s/$/\t>=\t'"${min_version}"'/
+              s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/
+              p
+              g
+              s/$/\t<=\t'"${max_version}"'/
+              :split_epoch
+              s/\(\s[0-9]\+\):\(\S\+\)$/\1\t\2/
+              t
+              s/\s\S\+$/\t0\0/
+            ' | \
             sed '
-              s|^| (from_base64("|
-              s|$|")),|
+              s/^/'"$(
+                printf '%s\t' \
+                  "${pkgname}" \
+                  "${arch}" \
+                  "${epoch}" \
+                  "${pkgver}" \
+                  "${pkgrel}" \
+                  "${build_arch}"
+              )"'/
             '
-          printf ' ("base-devel");\n'
-
-          printf 'INSERT IGNORE INTO `install_targets` (`name`)'
-          printf ' SELECT (`name`) FROM `%s` UNION' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends' | \
-            sed 's| UNION$|;\n|'
-
-          for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
-            case "${link}" in
-              'provides')
-                printf 'INSERT IGNORE INTO `install_target_providers` (`package`,`install_target`) SELECT'
-                printf ' `binary_packages`.`id`,`install_targets`.`id` FROM'
-              ;;
-              'makedepends'|'checkdepends'|'rundepends')
-                printf 'INSERT IGNORE INTO `dependencies` (`dependent`,`depending_on`,`dependency_type`) SELECT'
-                printf ' `binary_packages`.`id`,`install_targets`.`id`,`dependency_types`.`id` FROM'
-                printf ' `dependency_types` JOIN'
-              ;;
-            esac
-            printf ' `binary_packages`'
-            mysql_join_binary_packages_architectures
-            printf ' JOIN `install_targets`'
-            printf ' JOIN `%s`' "${link}"
-            printf ' ON `%s`.`name` = `install_targets`.`name`' "${link}"
-            printf ' WHERE'
-            if [ "${link}" = 'makedepends' ] || \
-              [ "${link}" = 'checkdepends' ] || \
-              [ "${link}" = 'rundepends' ]; then
-              printf ' `dependency_types`.`name` = "%s" AND' \
-                "${link%depends}"
-            fi
-            # TODO: identify binary_packages by id
-            printf ' `binary_packages`.`%s` = from_base64("%s") AND' \
-              'epoch' "${epoch}" \
-              'pkgver' "${pkgver}" \
-              'pkgrel' "${pkgrel}" \
-              'pkgname' "${pkgname}"
-            # we do not want to match the sub_pkgrel:
-            # a) it is tedious to do so (because it may be calculated
-            #    dynamically)
-            # b) it is not necessary to do so: if only the sub_pkgrel
-            #    changed, the dependencies and provided install_targets
-            #    should not have changed
-            printf ' `architectures`.`name` = from_base64("%s");\n' \
-              "${arch}"
-            # the repository is of no relevance: it hardly matters for
-            # the dependencies
           done
-
-          printf 'DROP TABLE `%s`;\n' \
-            'provides' \
-            'makedepends' \
-            'checkdepends' \
-            'rundepends'
-        } >> \
-          "${temp_dir}/add-install-targets-command"
-      done
+        rm "${temp_dir}/SRCINFO.tmp"
+      done | \
+      sort -u > \
+      "${temp_dir}/database-input"
 
     {
-      if [ -s "${temp_dir}/add-binary-packages-command" ]; then
-        cat "${temp_dir}/add-binary-packages-command"
-      fi
-      if [ -s "${temp_dir}/add-install-targets-command" ]; then
-        cat "${temp_dir}/add-install-targets-command"
-      fi
+      printf 'CREATE TEMPORARY TABLE `links`('
+        printf '`pkgname` VARCHAR(64),'
+        printf '`architecture` VARCHAR(16),'
+        printf '`epoch` MEDIUMINT,'
+        printf '`pkgver` VARCHAR(64),'
+        printf '`pkgrel` MEDIUMINT,'
+        printf '`build_architecture` VARCHAR(16),'
+        printf '`type` VARCHAR(16),'
+        printf '`install_target_name` VARCHAR(128),'
+        printf '`version_relation` VARCHAR(2),'
+        printf '`version_epoch` MEDIUMINT,'
+        printf '`version_version` VARCHAR(64)'
+      printf ');\n'
+      printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `links`;\n' \
+        "${temp_dir}/database-input"
+
+      printf 'SET @sub_pkgrel=('
+        printf 'SELECT MAX(`binary_packages`.`sub_pkgrel`)'
+        printf ' FROM `links`'
+        printf ' JOIN `architectures`'
+        printf ' ON `architectures`.`name`=`links`.`architecture`'
+        mysql_join_architectures_binary_packages
+        printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+          'pkgname' 'pkgname' \
+          'epoch' 'epoch' \
+          'pkgver' 'pkgver' \
+          'pkgrel' 'pkgrel'
+      printf ');\n'
+
+      printf 'INSERT IGNORE INTO `build_assignments` ('
+        printf '`package_source`,'
+        printf '`architecture`'
+      printf ') SELECT'
+      printf ' %s,`architectures`.`id`' \
+        "${package_source_id}"
+      printf ' FROM `links`'
+      printf ' JOIN `architectures`'
+      printf ' ON `architectures`.`name`=`links`.`build_architecture`'
+      printf ' WHERE `links`.`type`=`arch`;\n'
+
+      printf 'INSERT IGNORE INTO `binary_packages` ('
+        printf '`build_assignment`,'
+        printf '`pkgname`,'
+        printf '`epoch`,'
+        printf '`pkgver`,'
+        printf '`pkgrel`,'
+        printf '`sub_pkgrel`,'
+        printf '`architecture`'
+      printf ') SELECT'
+      printf ' `build_assignments`.`id`,'
+      printf '`links`.`%s`,' \
+        'pkgname' \
+        'epoch' \
+        'pkgver' \
+        'pkgrel'
+      printf '@sub_pkgrel,'
+      printf '`architectures`.`id`'
+      printf ' FROM `links`'
+      printf ' JOIN `architectures`'
+      printf ' ON `links`.`architecture`=`architectures`.`name`'
+      printf ' JOIN `architectures` AS `ba_a`'
+      printf ' ON `ba_a`.`name`=`links`.`build_architecture`'
+      mysql_join_architectures_build_assignments 'ba_a'
+      printf ' WHERE `links`.`type`=`arch`;\n'
+
+      printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
+      printf ' `package`'
+      printf ' `repository`'
+      printf ') SELECT'
+      printf ' `binary_packages`.`id`,'
+      printf '%s' \
+        "${current_repository_id}"
+      printf ' FROM `links`'
+      printf ' JOIN `architectures`'
+      printf ' ON `architectures`.`name`=`links`.`architecture`'
+      mysql_join_architectures_binary_packages
+      printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel'
+      printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+        'pkgname' 'pkgname' \
+        'epoch' 'epoch' \
+        'pkgver' 'pkgver' \
+        'pkgrel' 'pkgrel'
+      printf ';\n'
+
+      printf 'INSERT IGNORE INTO `install_targets` (`name`)'
+      printf ' SELECT `links`.`install_target_name`'
+      printf ' FROM `links`;\n'
+
+      printf 'INSERT IGNORE INTO `versions` (`epoch`,`version`)'
+      printf ' SELECT'
+      printf ' `links`.`version_epoch`,'
+      printf '`links`.`version_version`'
+      printf ' FROM `links`;\n'
+
+      for link in 'provides' 'makedepends' 'checkdepends' 'rundepends'; do
+        case "${link}" in
+          'provides')
+            printf 'INSERT IGNORE INTO `install_target_providers` ('
+              printf '`package`,'
+              printf '`install_target`,'
+              printf '`version`'
+            printf ') SELECT'
+            printf ' `binary_packages`.`id`,'
+            printf '`install_targets`.`id`,'
+            printf '`versions`.`id`'
+          ;;
+          'makedepends'|'checkdepends'|'rundepends')
+            printf 'INSERT IGNORE INTO `dependencies` ('
+              printf '`dependent`,'
+              printf '`depending_on`,'
+              printf '`dependency_type`,'
+              printf '`version`,'
+              printf '`version_relation`,'
+            printf ') SELECT'
+            printf ' `binary_packages`.`id`,'
+            printf '`install_targets`.`id`,'
+            printf '`dependency_types`.`id`,'
+            printf '`versions`.`id`,'
+            printf '`links`.`version_relation`,'
+          ;;
+        esac
+        printf ' FROM `links`'
+        printf ' JOIN `architectures`'
+        printf ' ON `architectures`.`name`=`links`.`architecture`'
+        mysql_join_architectures_binary_packages
+        printf ' AND `binary_packages`.`sub_pkgrel`=@sub_pkgrel'
+        printf ' AND `binary_packages`.`%s`=`links`.`%s`' \
+          'pkgname' 'pkgname' \
+          'epoch' 'epoch' \
+          'pkgver' 'pkgver' \
+          'pkgrel' 'pkgrel'
+        printf ' JOIN `install_targets`'
+        printf ' ON `install_targets`.`name`=`links`.`install_target_name`'
+        printf ' JOIN `versions`'
+        printf ' ON `versions`.`epoch`=`links`.`install_target_epoch`'
+        printf ' AND `versions`.`version`=`links`.`install_target_version`'
+        if [ "${link}" = 'makedepends' ] || \
+          [ "${link}" = 'checkdepends' ] || \
+          [ "${link}" = 'rundepends' ]; then
+          printf ' JOIN `dependency_types`'
+          printf ' ON `dependency_types`.`name`="%s"' \
+            "${link%depends}"
+        fi
+        printf ' WHERE `links`.`type`="%s";\n' \
+          "${link}"
+      done
+      printf 'DROP TEMPORARY TABLE `links`;\n'
     } | \
       mysql_run_query