summaryrefslogtreecommitdiff
path: root/bin/db-update
diff options
context:
space:
mode:
authorErich Eckner <git@eckner.net>2019-04-29 12:24:56 +0200
committerErich Eckner <git@eckner.net>2019-04-29 12:24:56 +0200
commitda7a6f8dde22e83739fcc9e31d1aec30a827dbf5 (patch)
treedf7adea29dd08f5938223f7bc7e7a435c900adfe /bin/db-update
parentfcee643710f1fb2dcd035a7328badf24c06d6d99 (diff)
downloadbuilder-da7a6f8dde22e83739fcc9e31d1aec30a827dbf5.tar.xz
bin/db-update: operate on a per-arch and per-stability basis
Diffstat (limited to 'bin/db-update')
-rwxr-xr-xbin/db-update707
1 files changed, 366 insertions, 341 deletions
diff --git a/bin/db-update b/bin/db-update
index 9e3bfd8..6768518 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -162,372 +162,397 @@ trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
export TMPDIR="${tmp_dir}"
-# shellcheck disable=SC2154
-for source_stability in \
- "${repository_stability_ids__testing}" \
- "${repository_stability_ids__staging}"; do
- find "${tmp_dir}" -xdev -mindepth 1 -delete
- # shellcheck disable=SC2016
- {
- # Note, that we do not need to check that two repositories a and b
- # are for the same architecture if we joined them via
- # repository_moves, because only repositories with the same
- # architectures should be listed there.
- if [ -n "${force_pkgs}" ]; then
- printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
- 'moveable' 'replaced'
- printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
- printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
- printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
- printf ' SELECT '
- printf '`binary_packages_in_repositories`.`id`,NULL'
- printf ' FROM `binary_packages_in_repositories`'
- mysql_join_binary_packages_in_repositories_repositories
- mysql_join_repositories_architectures
- mysql_join_binary_packages_in_repositories_binary_packages
- printf ' WHERE CONCAT('
- printf '`architectures`.`name`,"/",'
- printf '`repositories`.`name`,"/",'
- printf '`binary_packages`.`pkgname`'
- printf ') IN ('
- # shellcheck disable=SC2086
- printf 'from_base64("%s"),' \
- ${force_pkgs} | \
- sed 's/,$/);\n/'
- printf 'DELETE `moveable_bpir`'
+# shellcheck disable=SC2016
+{
+ printf 'SELECT DISTINCT'
+ printf ' `repositories`.`architecture`,'
+ printf '`repositories`.`stability`'
+ printf ' FROM `repositories`'
+ printf ' JOIN `repository_moves`'
+ printf ' ON `repository_moves`.`from_repository`=`repositories`.`id`'
+ printf ' WHERE `repositories`.`is_on_master_mirror`'
+ printf ' ORDER BY `repositories`.`stability`,'
+ printf 'SHA2('
+ printf 'CONCAT('
+ printf '`repositories`.`architecture`,'
+ printf '`repositories`.`stability`,'
+ printf 'NOW()'
+ printf '),'
+ printf '256'
+ printf ')'
+} | \
+ mysql_run_query | \
+ while read -r arch_id source_stability; do
+ find "${tmp_dir}" -xdev -mindepth 1 -delete
+
+ {
+ # Note, that we do not need to check that two repositories a and b
+ # are for the same architecture if we joined them via
+ # repository_moves, because only repositories with the same
+ # architectures should be listed there.
+ if [ -n "${force_pkgs}" ]; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
+ printf ' SELECT '
+ printf '`binary_packages_in_repositories`.`id`,NULL'
+ printf ' FROM `binary_packages_in_repositories`'
+ mysql_join_binary_packages_in_repositories_repositories
+ mysql_join_repositories_architectures
+ mysql_join_binary_packages_in_repositories_binary_packages
+ printf ' WHERE CONCAT('
+ printf '`architectures`.`name`,"/",'
+ printf '`repositories`.`name`,"/",'
+ printf '`binary_packages`.`pkgname`'
+ printf ') IN ('
+ # shellcheck disable=SC2086
+ printf 'from_base64("%s"),' \
+ ${force_pkgs} | \
+ sed 's/,$/);\n/'
+ printf 'DELETE `moveable_bpir`'
+ printf ' FROM `moveable_bpir`'
+ printf ' JOIN `binary_packages_in_repositories`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
+ mysql_join_binary_packages_in_repositories_binary_packages
+ mysql_join_binary_packages_in_repositories_repositories
+ printf ' WHERE `repositories`.`stability`!=%s' \
+ "${source_stability}"
+ printf ' OR `repositories`.`architecture`!=%s;\n' \
+ "${arch_id}"
+ printf 'UPDATE `moveable_bpir`'
+ printf ' JOIN `binary_packages_in_repositories`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
+ mysql_join_binary_packages_in_repositories_binary_packages
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
+ printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n'
+ printf 'DELETE FROM `moveable_bpir`'
+ printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n'
+ printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
+ printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`'
+ printf ' FROM `moveable_bpir`'
+ printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`'
+ printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`'
+ mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp'
+ printf ' JOIN `binary_packages`'
+ printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
+ mysql_join_binary_packages_binary_packages_in_repositories
+ mysql_join_binary_packages_in_repositories_repositories
+ printf ' JOIN `repositories` AS `m_to_r`'
+ printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`'
+ printf ' AND `repositories`.`stability`=`m_to_r`.`stability`'
+ printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n'
+ elif ${progressive}; then
+ printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
+ 'moveable' 'replaced'
+ printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+
+ # packages that replace broken packages
+ printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
+ printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`'
+ printf ' FROM `binary_packages_in_repositories` AS `new_bpir`'
+ mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp'
+ mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r'
+ printf ' AND `from_r`.`stability`=%s' \
+ "${source_stability}"
+ printf ' AND `from_r`.`architecture`=%s' \
+ "${arch_id}"
+ printf ' AND `from_r`.`is_on_master_mirror`'
+ mysql_join_binary_packages_build_assignments 'new_bp'
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`'
+ printf ' JOIN `repositories` AS `to_r`'
+ printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`'
+ printf ' JOIN `binary_packages` AS `old_bp`'
+ printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`'
+ mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir'
+ mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r'
+ printf ' AND `old_r`.`stability`=`to_r`.`stability`'
+ printf ' AND `old_r`.`architecture`=`to_r`.`architecture`'
+ mysql_join_binary_packages_dependencies 'old_bp'
+ mysql_join_dependencies_dependency_types
+ printf ' AND `dependency_types`.`relevant_for_binary_packages`'
+ mysql_join_dependencies_versions '' 'd_v'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT 1'
+ printf ' FROM `install_target_providers`'
+ mysql_join_install_target_providers_binary_packages '' 'itp_bp'
+ mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir'
+ mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r'
+ mysql_join_install_target_providers_versions '' 'itp_v'
+ printf ' JOIN `repository_stability_relations` AS `rsr_a`'
+ printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`'
+ printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
+ printf ' AND'
+ mysql_query_ordering_correct \
+ '`d_v`.`order`' \
+ '`itp_v`.`order`' \
+ '`dependencies`.`version_relation`'
+ printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`'
+ printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`'
+ # this is the least stable package of stability at least to_r.stability
+ printf ' AND NOT EXISTS ('
+ printf 'SELECT 1'
+ printf ' FROM `binary_packages` AS `subst_bp`'
+ mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir'
+ mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r'
+ printf ' JOIN `repository_stability_relations` AS `rsr_b`'
+ printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`'
+ printf ' JOIN `repository_stability_relations` AS `rsr_c`'
+ printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`'
+ printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`'
+ printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`'
+ printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`'
+ printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`'
+ printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`'
+ printf ')'
+ printf ');\n'
+
+ printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
+ printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`'
+ printf ' FROM `replaced_bpir`'
+ printf ' JOIN `binary_packages_in_repositories`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`'
+ printf ';\n'
+
+ # packages which replace nothing
+ printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
+ printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`'
+ printf ' FROM `binary_packages_in_repositories`'
+ mysql_join_binary_packages_in_repositories_binary_packages
+ mysql_join_binary_packages_in_repositories_repositories
+ printf ' AND `repositories`.`is_on_master_mirror`'
+ printf ' AND `repositories`.`stability`=%s' \
+ "${source_stability}"
+ printf ' AND `repositories`.`architecture`=%s' \
+ "${arch_id}"
+ mysql_join_binary_packages_build_assignments
+ mysql_join_build_assignments_package_sources
+ mysql_join_package_sources_upstream_repositories
+ mysql_join_upstream_repositories_repository_moves
+ printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
+ printf ' JOIN `repositories` as `to_r`'
+ printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`'
+ printf ' WHERE NOT EXISTS ('
+ printf 'SELECT 1'
+ printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`'
+ mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp'
+ mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r'
+ printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
+ printf ' AND `repl_r`.`stability`=`to_r`.`stability`'
+ printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`'
+ printf ');\n'
+ else
+ printf 'CALL calculate_maximal_moveable_set(%s,%s);\n' \
+ "${arch_id}" \
+ "${source_stability}"
+ fi
+
+ printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `moveable_bpir`.`to_repository`'
+ printf ' FROM `moveable_bpir`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages_in_repositories`.`repository`'
printf ' FROM `moveable_bpir`'
printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
- mysql_join_binary_packages_in_repositories_binary_packages
- mysql_join_binary_packages_in_repositories_repositories
- printf ' WHERE `repositories`.`stability`!=%s;\n' \
- "${source_stability}"
- printf 'UPDATE `moveable_bpir`'
+ printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
+ printf 'INSERT IGNORE INTO `rps` (`id`)'
+ printf ' SELECT `binary_packages_in_repositories`.`repository`'
+ printf ' FROM `replaced_bpir`'
printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `binary_packages_in_repositories`.`id`=`moveable_bpir`.`id`'
- mysql_join_binary_packages_in_repositories_binary_packages
- mysql_join_binary_packages_build_assignments
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- mysql_join_upstream_repositories_repository_moves
- printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
- printf ' SET `moveable_bpir`.`to_repository`=`repository_moves`.`to_repository`;\n'
- printf 'DELETE FROM `moveable_bpir`'
- printf ' WHERE `moveable_bpir`.`to_repository` IS NULL;\n'
- printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
- printf ' SELECT `binary_packages_in_repositories`.`id`,`moveable_bpir`.`id`'
+ printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
+
+ printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`'
+ printf ' FROM `rps`'
+ printf ' JOIN `repositories`'
+ printf ' ON `rps`.`id`=`repositories`.`id`'
+ mysql_join_repositories_architectures
+ printf ';\n'
+
+ printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`'
+ printf ' FROM `moveable_bpir`;\n'
+
+ printf 'SELECT "mv",'
+ mysql_package_name_query
+ printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`'
printf ' FROM `moveable_bpir`'
- printf ' JOIN `binary_packages_in_repositories` AS `subst_bpir`'
- printf ' ON `moveable_bpir`.`id`=`subst_bpir`.`id`'
- mysql_join_binary_packages_in_repositories_binary_packages 'subst_bpir' 'subst_bp'
- printf ' JOIN `binary_packages`'
- printf ' ON `binary_packages`.`pkgname`=`subst_bp`.`pkgname`'
- mysql_join_binary_packages_binary_packages_in_repositories
- mysql_join_binary_packages_in_repositories_repositories
- printf ' JOIN `repositories` AS `m_to_r`'
- printf ' ON `moveable_bpir`.`to_repository`=`m_to_r`.`id`'
- printf ' AND `repositories`.`stability`=`m_to_r`.`stability`'
- printf ' AND `repositories`.`architecture`=`m_to_r`.`architecture`;\n'
- elif ${progressive}; then
- printf 'DROP TEMPORARY TABLE IF EXISTS `%s_bpir`;\n' \
- 'moveable' 'replaced'
- printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, `replaced_by` BIGINT, UNIQUE KEY (`id`));\n'
- printf 'CREATE TEMPORARY TABLE `moveable_bpir` (`id` BIGINT, `to_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
-
- # packages that replace broken packages
- printf 'INSERT IGNORE INTO `replaced_bpir` (`id`,`replaced_by`)'
- printf ' SELECT `old_bpir`.`id`,`new_bpir`.`id`'
- printf ' FROM `binary_packages_in_repositories` AS `new_bpir`'
- mysql_join_binary_packages_in_repositories_binary_packages 'new_bpir' 'new_bp'
- mysql_join_binary_packages_in_repositories_repositories 'new_bpir' 'from_r'
- printf ' AND `from_r`.`stability`=%s' \
- "${source_stability}"
- printf ' AND `from_r`.`is_on_master_mirror`'
- mysql_join_binary_packages_build_assignments 'new_bp'
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- mysql_join_upstream_repositories_repository_moves
- printf ' AND `repository_moves`.`from_repository`=`from_r`.`id`'
- printf ' JOIN `repositories` AS `to_r`'
- printf ' ON `repository_moves`.`to_repository`=`to_r`.`id`'
- printf ' JOIN `binary_packages` AS `old_bp`'
- printf ' ON `new_bp`.`pkgname`=`old_bp`.`pkgname`'
- mysql_join_binary_packages_binary_packages_in_repositories 'old_bp' 'old_bpir'
- mysql_join_binary_packages_in_repositories_repositories 'old_bpir' 'old_r'
- printf ' AND `old_r`.`stability`=`to_r`.`stability`'
- printf ' AND `old_r`.`architecture`=`to_r`.`architecture`'
- mysql_join_binary_packages_dependencies 'old_bp'
- mysql_join_dependencies_dependency_types
- printf ' AND `dependency_types`.`relevant_for_binary_packages`'
- mysql_join_dependencies_versions '' 'd_v'
- printf ' WHERE NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `install_target_providers`'
- mysql_join_install_target_providers_binary_packages '' 'itp_bp'
- mysql_join_binary_packages_binary_packages_in_repositories 'itp_bp' 'itp_bpir'
- mysql_join_binary_packages_in_repositories_repositories 'itp_bpir' 'itp_r'
- mysql_join_install_target_providers_versions '' 'itp_v'
- printf ' JOIN `repository_stability_relations` AS `rsr_a`'
- printf ' ON `rsr_a`.`more_stable`=`itp_r`.`stability`'
- printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
- printf ' AND'
- mysql_query_ordering_correct \
- '`d_v`.`order`' \
- '`itp_v`.`order`' \
- '`dependencies`.`version_relation`'
- printf ' AND `itp_r`.`architecture`=`old_r`.`architecture`'
- printf ' AND `rsr_a`.`less_stable`=`old_r`.`stability`'
- # this is the least stable package of stability at least to_r.stability
- printf ' AND NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `binary_packages` AS `subst_bp`'
- mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bpir'
- mysql_join_binary_packages_in_repositories_repositories 'subst_bpir' 'subst_r'
- printf ' JOIN `repository_stability_relations` AS `rsr_b`'
- printf ' ON `rsr_b`.`less_stable`=`subst_r`.`stability`'
- printf ' JOIN `repository_stability_relations` AS `rsr_c`'
- printf ' ON `rsr_c`.`more_stable`=`subst_r`.`stability`'
- printf ' WHERE `subst_bp`.`pkgname`=`itp_bp`.`pkgname`'
- printf ' AND `subst_bp`.`id`!=`itp_bp`.`id`'
- printf ' AND `rsr_b`.`more_stable`=`itp_r`.`stability`'
- printf ' AND `rsr_c`.`less_stable`=`old_r`.`stability`'
- printf ' AND `subst_r`.`architecture`=`itp_r`.`architecture`'
- printf ')'
- printf ');\n'
-
- printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
- printf ' SELECT `replaced_bpir`.`replaced_by`,`binary_packages_in_repositories`.`repository`'
- printf ' FROM `replaced_bpir`'
printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`'
+ printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`'
+ mysql_join_binary_packages_in_repositories_binary_packages
+ mysql_join_binary_packages_in_repositories_repositories
+ mysql_join_binary_packages_architectures
+ mysql_join_repositories_architectures '' 'r_a'
+ printf ' JOIN `repositories` AS `new_repo`'
+ printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`'
+ mysql_join_repositories_architectures 'new_repo' 'new_r_a'
printf ';\n'
- # packages which replace nothing
- printf 'INSERT IGNORE INTO `moveable_bpir` (`id`,`to_repository`)'
- printf ' SELECT `binary_packages_in_repositories`.`id`,`repository_moves`.`to_repository`'
- printf ' FROM `binary_packages_in_repositories`'
+ printf 'SELECT "rm.id",`replaced_bpir`.`id`'
+ printf ' FROM `replaced_bpir`;\n'
+
+ printf 'SELECT "rm",'
+ mysql_package_name_query
+ printf ',`r_a`.`name`,`repositories`.`name`'
+ printf ' FROM `replaced_bpir`'
+ printf ' JOIN `binary_packages_in_repositories`'
+ printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`'
mysql_join_binary_packages_in_repositories_binary_packages
mysql_join_binary_packages_in_repositories_repositories
- printf ' AND `repositories`.`is_on_master_mirror`'
- printf ' AND `repositories`.`stability`=%s' \
- "${source_stability}"
- mysql_join_binary_packages_build_assignments
- mysql_join_build_assignments_package_sources
- mysql_join_package_sources_upstream_repositories
- mysql_join_upstream_repositories_repository_moves
- printf ' AND `repository_moves`.`from_repository`=`binary_packages_in_repositories`.`repository`'
- printf ' JOIN `repositories` as `to_r`'
- printf ' ON `to_r`.`id`=`repository_moves`.`to_repository`'
- printf ' WHERE NOT EXISTS ('
- printf 'SELECT 1'
- printf ' FROM `binary_packages_in_repositories` AS `repl_bpir`'
- mysql_join_binary_packages_in_repositories_binary_packages 'repl_bpir' 'repl_bp'
- mysql_join_binary_packages_in_repositories_repositories 'repl_bpir' 'repl_r'
- printf ' WHERE `repl_bp`.`pkgname`=`binary_packages`.`pkgname`'
- printf ' AND `repl_r`.`stability`=`to_r`.`stability`'
- printf ' AND `repl_r`.`architecture`=`to_r`.`architecture`'
- printf ');\n'
- else
- printf 'CALL calculate_maximal_moveable_set(%s);\n' \
+ mysql_join_binary_packages_architectures
+ mysql_join_repositories_architectures '' 'r_a'
+ printf ';\n'
+ } | \
+ mysql_run_query | \
+ tr '\t' ' ' | \
+ grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
+ while read -r what content; do
+ printf '%s\n' "${content}" >> \
+ "${tmp_dir}/${what}"
+ done
+
+ if [ ! -s "${tmp_dir}/repositories" ]; then
+ >&2 printf 'Nothing to move here (%s,%s).\n' \
+ "${arch_id}" \
"${source_stability}"
+ continue
fi
- printf 'CREATE TEMPORARY TABLE `rps` (`id` MEDIUMINT, UNIQUE INDEX (`id`));\n'
- printf 'INSERT IGNORE INTO `rps` (`id`)'
- printf ' SELECT `moveable_bpir`.`to_repository`'
- printf ' FROM `moveable_bpir`;\n'
- printf 'INSERT IGNORE INTO `rps` (`id`)'
- printf ' SELECT `binary_packages_in_repositories`.`repository`'
- printf ' FROM `moveable_bpir`'
- printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
- printf 'INSERT IGNORE INTO `rps` (`id`)'
- printf ' SELECT `binary_packages_in_repositories`.`repository`'
- printf ' FROM `replaced_bpir`'
- printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`;\n'
-
- printf 'SELECT "repositories",`architectures`.`name`,`repositories`.`name`'
- printf ' FROM `rps`'
- printf ' JOIN `repositories`'
- printf ' ON `rps`.`id`=`repositories`.`id`'
- mysql_join_repositories_architectures
- printf ';\n'
-
- printf 'SELECT "mv.id",`moveable_bpir`.`id`,`moveable_bpir`.`to_repository`'
- printf ' FROM `moveable_bpir`;\n'
-
- printf 'SELECT "mv",'
- mysql_package_name_query
- printf ',`r_a`.`name`,`repositories`.`name`,`new_r_a`.`name`,`new_repo`.`name`'
- printf ' FROM `moveable_bpir`'
- printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `moveable_bpir`.`id`=`binary_packages_in_repositories`.`id`'
- mysql_join_binary_packages_in_repositories_binary_packages
- mysql_join_binary_packages_in_repositories_repositories
- mysql_join_binary_packages_architectures
- mysql_join_repositories_architectures '' 'r_a'
- printf ' JOIN `repositories` AS `new_repo`'
- printf ' ON `new_repo`.`id`=`moveable_bpir`.`to_repository`'
- mysql_join_repositories_architectures 'new_repo' 'new_r_a'
- printf ';\n'
-
- printf 'SELECT "rm.id",`replaced_bpir`.`id`'
- printf ' FROM `replaced_bpir`;\n'
-
- printf 'SELECT "rm",'
- mysql_package_name_query
- printf ',`r_a`.`name`,`repositories`.`name`'
- printf ' FROM `replaced_bpir`'
- printf ' JOIN `binary_packages_in_repositories`'
- printf ' ON `replaced_bpir`.`id`=`binary_packages_in_repositories`.`id`'
- mysql_join_binary_packages_in_repositories_binary_packages
- mysql_join_binary_packages_in_repositories_repositories
- mysql_join_binary_packages_architectures
- mysql_join_repositories_architectures '' 'r_a'
- printf ';\n'
- } | \
- mysql_run_query | \
- tr '\t' ' ' | \
- grep '^\(repositories\|\(rm\|mv\)\(\.id\)\?\) ' | \
- while read -r what content; do
- printf '%s\n' "${content}" >> \
- "${tmp_dir}/${what}"
- done
-
- if [ ! -s "${tmp_dir}/repositories" ]; then
- >&2 printf 'Nothing to move here (%s).\n' "${source_stability}"
- continue
- fi
-
- touch \
- "${tmp_dir}/mv" \
- "${tmp_dir}/mv.id" \
- "${tmp_dir}/rm" \
- "${tmp_dir}/rm.id"
-
- # shellcheck disable=SC2086
- for s in "${tmp_dir}/"*; do
- sort -u "${s}" | \
- sponge "${s}"
- done
+ touch \
+ "${tmp_dir}/mv" \
+ "${tmp_dir}/mv.id" \
+ "${tmp_dir}/rm" \
+ "${tmp_dir}/rm.id"
- # receive the repository databases from the master mirror
- mkdir "${tmp_dir}/dbs"
- while read -r arch repo; do
- mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
# shellcheck disable=SC2086
- failsafe_rsync \
- "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
- "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
- "${tmp_dir}/dbs/${arch}/${repo}/"
- done < \
- "${tmp_dir}/repositories"
-
- # remove to-be-deleted packages
- # shellcheck disable=SC2094
- cut -d' ' -f2,3 < \
- "${tmp_dir}/rm" | \
- sort -u | \
- while read -r arch repo; do
- grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
- sed '
- s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
- ' | \
- xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
+ for s in "${tmp_dir}/"*; do
+ sort -u "${s}" | \
+ sponge "${s}"
done
- # copy and delete moved packages
- # shellcheck disable=SC2094
- cut -d' ' -f2,3,4,5 < \
- "${tmp_dir}/mv" | \
- sort -u | \
- while read -r from_arch from_repo to_arch to_repo; do
- grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
- sed '
- s/-[^-]\+\( \S\+\)\{4\}$//
- ' | \
- xargs -r "${base_dir}/bin/repo-copy" \
- "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
- "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
- grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
- sed '
- s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
- ' | \
- xargs -r repo-remove -q \
- "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
- done
+ # receive the repository databases from the master mirror
+ mkdir "${tmp_dir}/dbs"
+ while read -r arch repo; do
+ mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
+ # shellcheck disable=SC2086
+ failsafe_rsync \
+ "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
+ "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
+ "${tmp_dir}/dbs/${arch}/${repo}/"
+ done < \
+ "${tmp_dir}/repositories"
+
+ # remove to-be-deleted packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2,3 < \
+ "${tmp_dir}/rm" | \
+ sort -u | \
+ while read -r arch repo; do
+ grep " $(str_to_regex "${arch} ${repo}")\$" "${tmp_dir}/rm" | \
+ sed '
+ s/\(-[^-]\+\)\{3\} \S\+ \S\+$//
+ ' | \
+ xargs -r repo-remove -q "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db.tar.gz"
+ done
+
+ # copy and delete moved packages
+ # shellcheck disable=SC2094
+ cut -d' ' -f2,3,4,5 < \
+ "${tmp_dir}/mv" | \
+ sort -u | \
+ while read -r from_arch from_repo to_arch to_repo; do
+ grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/-[^-]\+\( \S\+\)\{4\}$//
+ ' | \
+ xargs -r "${base_dir}/bin/repo-copy" \
+ "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz" \
+ "${tmp_dir}/dbs/${to_arch}/${to_repo}/${to_repo}.db.tar.gz"
+ grep " $(str_to_regex "${from_arch} ${from_repo} ${to_arch} ${to_repo}")\$" "${tmp_dir}/mv" | \
+ sed '
+ s/\(-[^-]\+\)\{3\}\( \S\+\)\{4\}$//
+ ' | \
+ xargs -r repo-remove -q \
+ "${tmp_dir}/dbs/${from_arch}/${from_repo}/${from_repo}.db.tar.gz"
+ done
+
+ # create real file names of packages, because
+ # mysql_query_and_delete_unneeded_binary_packages does so, too
+ sed -i '
+ s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
+ ' "${tmp_dir}/rm"
- # create real file names of packages, because
- # mysql_query_and_delete_unneeded_binary_packages does so, too
- sed -i '
- s,^\(\S\+\) \(\S\+\) \(\S\+\)$,\2/\3/\1,
- ' "${tmp_dir}/rm"
+ # somewhat inaccurate
+ if ! ${no_action}; then
+ {
+ printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
+ printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
+ printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
+ "${tmp_dir}/mv.id" 'moved_bpir' \
+ "${tmp_dir}/rm.id" 'replaced_bpir'
+ printf 'DELETE `binary_packages_in_repositories`'
+ printf ' FROM `binary_packages_in_repositories`'
+ printf ' JOIN `replaced_bpir`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
+ mysql_query_and_delete_unneeded_binary_packages
+ printf 'UPDATE `binary_packages_in_repositories`'
+ printf ' JOIN `moved_bpir`'
+ printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
+ printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
+ printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
+ printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
+ } | \
+ mysql_run_query | \
+ sort -u >> \
+ "${tmp_dir}/rm"
+ fi
- # somewhat inaccurate
- if ! ${no_action}; then
- # shellcheck disable=SC2016
+ # move the packages remotely via sftp
{
- printf 'CREATE TEMPORARY TABLE `replaced_bpir` (`id` BIGINT, UNIQUE KEY (`id`));\n'
- printf 'CREATE TEMPORARY TABLE `moved_bpir` (`id` BIGINT, `new_repository` MEDIUMINT, UNIQUE KEY (`id`));\n'
- printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s` COLUMNS TERMINATED BY " ";\n' \
- "${tmp_dir}/mv.id" 'moved_bpir' \
- "${tmp_dir}/rm.id" 'replaced_bpir'
- printf 'DELETE `binary_packages_in_repositories`'
- printf ' FROM `binary_packages_in_repositories`'
- printf ' JOIN `replaced_bpir`'
- printf ' ON `binary_packages_in_repositories`.`id`=`replaced_bpir`.`id`;\n'
- mysql_query_and_delete_unneeded_binary_packages
- printf 'UPDATE `binary_packages_in_repositories`'
- printf ' JOIN `moved_bpir`'
- printf ' ON `binary_packages_in_repositories`.`id`=`moved_bpir`.`id`'
- printf ' SET `binary_packages_in_repositories`.`repository`=`moved_bpir`.`new_repository`,'
- printf '`binary_packages_in_repositories`.`last_moved`=NOW()'
- printf ' WHERE `binary_packages_in_repositories`.`repository`!=`moved_bpir`.`new_repository`;\n'
+ sed '
+ s/^/rm "/
+ s/$/"/
+ p
+ s/"$/.sig"/
+ ' "${tmp_dir}/rm"
+ sed '
+ s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
+ ' "${tmp_dir}/mv"
+ echo 'quit'
} | \
- mysql_run_query | \
- sort -u >> \
- "${tmp_dir}/rm"
- fi
+ if ${no_action}; then
+ sed 's|^|sftp: |'
+ else
+ failsafe_sftp
+ fi
- # move the packages remotely via sftp
- {
- sed '
- s/^/rm "/
- s/$/"/
- p
- s/"$/.sig"/
- ' "${tmp_dir}/rm"
- sed '
- s,^\(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\) \(\S\+\)$,rename "\2/\3/\1" "\4/\5/\1"\nrename "\2/\3/\1.sig" "\4/\5/\1.sig",
- ' "${tmp_dir}/mv"
- echo 'quit'
- } | \
if ${no_action}; then
- sed 's|^|sftp: |'
- else
- failsafe_sftp
+ continue
fi
- if ${no_action}; then
- continue
- fi
-
- # and push our local *.db.tar.gz via rsync
- while read -r arch repo; do
- recompress_gz \
- "${tmp_dir}" \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
- # shellcheck disable=SC2086
- failsafe_rsync \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
- "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
- "${master_mirror_rsync_directory}/${arch}/${repo}/"
- done < \
- "${tmp_dir}/repositories"
+ # and push our local *.db.tar.gz via rsync
+ while read -r arch repo; do
+ recompress_gz \
+ "${tmp_dir}" \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
+ # shellcheck disable=SC2086
+ failsafe_rsync \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
+ "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
+ "${master_mirror_rsync_directory}/${arch}/${repo}/"
+ done < \
+ "${tmp_dir}/repositories"
-done
+ done
trigger_mirror_refreshs