feature(parallel): run up to N jobs in parallel for repo clone/configure

Run up to N jobs in parallel. By default the number of jobs is equal to the
number of available processing units. For sequential processing this option
needs to be passed with 1.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

7 files changed, 71 insertions, 7 deletions

diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in
index b0a90e5..17f863f 100644
--- a/contrib/completion/bash/devtools.in
+++ b/contrib/completion/bash/devtools.in
@@ -268,17 +268,23 @@ _pkgctl_repo_clone_args=(
 	--switch
 	-u --unprivileged
 	--universe
+	-j --jobs
 	-h --help
 )
 _pkgctl_repo_clone_args__maintainer_opts() { :; }
 _pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; }
 _pkgctl_repo_clone_args__switch_opts() { :; }
+_pkgctl_repo_clone_args__jobs_opts() { :; }
+_pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
 _pkgctl_repo_clone_opts() { _devtools_completions_all_packages; }
 
 
 _pkgctl_repo_configure_args=(
+	-j --jobs
 	-h --help
 )
+_pkgctl_repo_configure_args__jobs_opts() { :; }
+_pkgctl_repo_configure_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
 _pkgctl_repo_configure_opts() { _filedir -d; }
 
 
diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in
index 45ffde3..240f781 100644
--- a/contrib/completion/zsh/_devtools.in
+++ b/contrib/completion/zsh/_devtools.in
@@ -110,11 +110,13 @@ _pkgctl_repo_clone_args=(
 	'(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:'
 	'--switch=[Switch the current working tree to a specified version]'
 	'--universe[Clone all existing packages, useful for cache warming]'
+	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:packages:_devtools_completions_all_packages'
 )
 
 _pkgctl_repo_configure_args=(
+	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:git_dir:_files -/'
 )
diff --git a/doc/man/pkgctl-repo-clone.1.asciidoc b/doc/man/pkgctl-repo-clone.1.asciidoc
index 487ec35..d6da062 100644
--- a/doc/man/pkgctl-repo-clone.1.asciidoc
+++ b/doc/man/pkgctl-repo-clone.1.asciidoc
@@ -32,6 +32,11 @@ Options
 	Switch to a specified version. The working tree and the index are updated to
 	match the version.
 
+*-j, --jobs* 'N'::
+	Run up to N jobs in parallel. By default the number of jobs is equal to the
+	number of available processing units. For sequential processing this option
+	needs to be passed with 1.
+
 *-h, --help*::
 	Show a help text
 
diff --git a/doc/man/pkgctl-repo-configure.1.asciidoc b/doc/man/pkgctl-repo-configure.1.asciidoc
index 4499ed6..1b07dc7 100644
--- a/doc/man/pkgctl-repo-configure.1.asciidoc
+++ b/doc/man/pkgctl-repo-configure.1.asciidoc
@@ -25,6 +25,11 @@ read-only HTTPS otherwise.
 Options
 -------
 
+*-j, --jobs* 'N'::
+	Run up to N jobs in parallel. By default the number of jobs is equal to the
+	number of available processing units. For sequential processing this option
+	needs to be passed with 1.
+
 *-h, --help*::
 	Show a help text
 
diff --git a/src/lib/common.sh b/src/lib/common.sh
index 24479eb..3d1ee56 100644
--- a/src/lib/common.sh
+++ b/src/lib/common.sh
@@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE
 export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
 
 # check if messages are to be printed using color
-if [[ -t 2 && "$TERM" != dumb ]]; then
+if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
 	colorize
 else
 	# shellcheck disable=2034
diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index 340aa69..a02d799 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() {
 		    -m, --maintainer=NAME  Clone all packages of the named maintainer
 		    --switch VERSION       Switch the current working tree to a specified version
 		    --universe             Clone all existing packages, useful for cache warming
+		    -j, --jobs N           Run up to N jobs in parallel (default: $(nproc))
 		    -h, --help             Show this help text
 
 		EXAMPLES
@@ -55,9 +56,11 @@ pkgctl_repo_clone() {
 	local MAINTAINER=
 	local VERSION=
 	local CONFIGURE_OPTIONS=()
-	local pkgbases
+	local jobs=
+	jobs=$(nproc)
 
 	# variables
+	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
 	local project_path
 
 	while (( $# )); do
@@ -97,6 +100,11 @@ pkgctl_repo_clone() {
 				CLONE_ALL=1
 				shift
 				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
 			--)
 				shift
 				break
@@ -142,12 +150,31 @@ pkgctl_repo_clone() {
 		stat_done
 	fi
 
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then
+		# force colors in parallel if parent process is colorized
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+		# assign command options
+		if [[ -n "${VERSION}" ]]; then
+			command+=" --switch '${VERSION}'"
+		fi
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then
+			die 'Failed to clone some packages, please check the output'
+			exit 1
+		fi
+		exit 0
+	fi
+
 	for pkgbase in "${pkgbases[@]}"; do
 		if [[ ! -d ${pkgbase} ]]; then
 			msg "Cloning ${pkgbase} ..."
 			project_path=$(gitlab_project_name_to_path "${pkgbase}")
 			remote_url="${GIT_REPO_BASE_URL}/${project_path}.git"
-			git clone --origin origin "${remote_url}" "${pkgbase}"
+			if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then
+				die 'failed to clone %s' "${pkgbase}"
+			fi
 		else
 			warning "Skip cloning ${pkgbase}: Directory exists"
 		fi
diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh
index 942876a..a5708a0 100644
--- a/src/lib/repo/configure.sh
+++ b/src/lib/repo/configure.sh
@@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() {
 		read-only HTTPS otherwise.
 
 		OPTIONS
+		    -j, --jobs N         Run up to N jobs in parallel (default: $(nproc))
 		    -h, --help           Show this help text
 
 		EXAMPLES
@@ -93,9 +94,12 @@ pkgctl_repo_configure() {
 	local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS}
 	local official=0
 	local proto=https
+	local jobs=
+	jobs=$(nproc)
 	local paths=()
 
 	# variables
+	local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
 	local path realpath pkgbase remote_url project_path
 	local PACKAGER GPGKEY packager_name packager_email
 
@@ -105,6 +109,11 @@ pkgctl_repo_configure() {
 				pkgctl_repo_configure_usage
 				exit 0
 				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
 			--)
 				shift
 				break
@@ -157,10 +166,21 @@ pkgctl_repo_configure() {
 		msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}"
 	fi
 
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+			die 'Failed to configure some packages, please check the output'
+			exit 1
+		fi
+		exit 0
+	fi
+
 	for path in "${paths[@]}"; do
 		if ! realpath=$(realpath -e "${path}"); then
-			error "No such directory: ${path}"
-			continue
+			die "No such directory: ${path}"
 		fi
 
 		pkgbase=$(basename "${realpath}")
@@ -168,8 +188,7 @@ pkgctl_repo_configure() {
 		msg "Configuring ${pkgbase}"
 
 		if [[ ! -d "${path}/.git" ]]; then
-			error "Not a Git repository: ${path}"
-			continue
+			die "Not a Git repository: ${path}"
 		fi
 
 		pushd "${path}" >/dev/null