cleanup and plan for stage4

12 files changed, 397 insertions, 15 deletions

diff --git a/README b/README
index 656a6a8..6c1ac66 100644
--- a/README
+++ b/README
@@ -219,6 +219,9 @@ su cross ./prepare_stage2_repo.sh
 # - libsecret, calls gtkdocize in autogen.sh
 # - systemd-sysvcompat: should be optional, I don't see why this should
 #   be in base
+# - nss: has big issues with hand-crafted assembler optimizations all
+#   over the place drawing in SSE2 as a requirement, used for
+#   building ca-certificates-mozilla only at the moment
 
 # systemd: goal is to have a libsystemd we can link against, but stage 3
 # will not have a running systemd init system!
@@ -226,7 +229,7 @@ su cross ./prepare_stage2_repo.sh
 su cross ./prepare_stage3_repo.sh
 
 # Build stage 3 on the target architecture and install it onto the
-# stage 1 system. Resulting artifacts get stored also back
+# stage 2 system. Resulting artifacts get stored also back
 # in $STAGE3_BUILD.
 
 # Most things compile just fine with 64 MB RAM, binutils ld gold needs more
@@ -242,5 +245,15 @@ su cross ./prepare_stage3_repo.sh
 # use vanilla PKGBUILDs as much as possible, also building documentation
 # and do all the testing.
 
+# We don't go to another build system for now till we can build the base
+# system without patched PKGBUILDs (at least not where not needed).
+
 # We also try to get a running systemd in this stage.
 
+su cross ./prepare_stage4_repo.sh
+
+# Build stage 4 on the target architecture and install it onto the
+# stage 3 system. Resulting artifacts get stored also back
+# in $STAGE4_BUILD.
+
+./build_stage4.sh
diff --git a/TODOS b/TODOS
index 2e40bcf..f4e3767 100644
--- a/TODOS
+++ b/TODOS
@@ -38,7 +38,9 @@ general bugs:
   works nevertheless
 - introduce a '-package' along to '(+)package' to delete a package at a
   given place (e.g. a shim)
-    
+- get rid of ADDITIONAL_PACKAGES, we actually parse pkgname and install
+  all mentioned packages directly
+  
 stage1 issues:
   - stage1: cdrom installs keyrings without having a gpg binary
   - 32 MB is not enough when installing packages => add a swap as first action!
diff --git a/build_stage1_package.sh b/build_stage1_package.sh
index 97a81ad..dacd590 100755
--- a/build_stage1_package.sh
+++ b/build_stage1_package.sh
@@ -52,7 +52,7 @@ if test "$(pacman --config "$STAGE1_CHROOT/etc/pacman.conf" -r "$STAGE1_CHROOT"
 			cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" .
 			;;
 		*)
-			print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
+			echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
 			exit 1
 	esac
 			
diff --git a/build_stage2_package.sh b/build_stage2_package.sh
index 2b58eff..ab06ba8 100755
--- a/build_stage2_package.sh
+++ b/build_stage2_package.sh
@@ -49,7 +49,7 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 			cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" .
 			;;
 		*)
-			print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
+			echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
 			exit 1
 	esac
 			
@@ -99,6 +99,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 	scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE2_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/.
 
 	# building the actual package
+
+	echo "Building $PACKAGE on target.."
 	
 	ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1
 	RES=$?
@@ -106,6 +108,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 	tail "$PACKAGE.log"
 
 	if test $RES = 0; then
+	
+		echo "Package $PACKAGE built sucessfully, installing on target.."
 
 		# copy to our package folder in the first stage chroot
 		
@@ -119,10 +123,10 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 		# there, packages seem to reappear in old versions)
 
 		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'
-			rm -rf /var/cache/pacman/pkg/*
-			rm -rf /packages/$TARGET_CPU/temp.db*
-			rm -rf /packages/$TARGET_CPU/temp.files*
-			repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz
+#			rm -rf /var/cache/pacman/pkg/*
+#			rm -rf /packages/$TARGET_CPU/temp.db*
+#			rm -rf /packages/$TARGET_CPU/temp.files*
+#			repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz
 		'"
 		
 		# install onto stage 1 system via pacman
@@ -130,7 +134,7 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 		if test "$FORCE_INSTALL"; then
 			FORCE="--force"
 		fi
-
+		
 		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'		
 			# TODO: broken [temp] repo
 			if test \"$ADDITIONAL_INSTALL_PACKAGE\" != \"\"; then
@@ -144,6 +148,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 		
 		# copy packages from target machine and replace our local version with it
 
+		echo "Salvaging build environment of $PACKAGE from target back to host.."
+
 		tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir)
 		trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 		
diff --git a/build_stage3_package.sh b/build_stage3_package.sh
index 2c2fdcb..c265212 100755
--- a/build_stage3_package.sh
+++ b/build_stage3_package.sh
@@ -16,7 +16,7 @@ PACKAGE=$1
 
 . "$SCRIPT_DIR/$TARGET_CPU-stage3/template/DESCR"
 
-#~ if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then
+if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then
 	echo "Building package $PACKAGE."
 
 	cd $STAGE3_BUILD || exit 1
@@ -49,7 +49,7 @@ PACKAGE=$1
 			cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" .
 			;;
 		*)
-			print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
+			echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
 			exit 1
 	esac
 			
@@ -174,6 +174,6 @@ PACKAGE=$1
 
 	cd $STAGE3_BUILD || exit 1
 
-#~ else
-	#~ echo "$PACKAGE exists."
-#~ fi
+else
+	echo "$PACKAGE exists."
+fi
diff --git a/build_stage4.sh b/build_stage4.sh
new file mode 100755
index 0000000..40c1adb
--- /dev/null
+++ b/build_stage4.sh
@@ -0,0 +1,121 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# build all packages for stage 4 using the target system with stage 3
+# packages.  packages will be installed with pacman onto the target
+# system once built sucessfully. The artifacts are also copied back
+# to the $STAGE4_PACKAGES to speed up rebuild of the state of the stage 4
+# system in case of destroying it.
+
+PACKAGES=""
+
+# Archlinux base, base-devel groups
+#~ bash
+#~ bzip2
+#~ coreutils
+#~ cryptsetup
+#~ device-mapper
+#~ dhcpcd
+#~ diffutils
+#~ e2fsprogs
+#~ file
+#~ filesystem
+#~ findutils
+#~ gawk
+#~ gcc-libs
+#~ gcc-libs
+#~ gettext
+#~ glibc
+#~ grep
+#~ gzip
+#~ inetutils
+#~ iproute2
+#~ iputils
+#~ jfsutils
+#~ less
+#~ licenses
+#~ linux
+#~ logrotate
+#~ lvm2
+#~ man-db
+#~ man-pages
+#~ mdadm
+#~ nano
+#~ netctl
+#~ pacman
+#~ pciutils
+#~ pcmciautils
+#~ perl
+#~ procps-ng
+#~ psmisc
+#~ reiserfsprogs
+#~ s-nail
+#~ sed
+#~ shadow
+#~ sysfsutils
+#~ systemd-sysvcompat
+#~ tar
+#~ texinfo
+#~ usbutils
+#~ util-linux
+#~ vi
+#~ which
+#~ xfsprogs
+
+#~ autoconf
+#~ automake
+#~ binutils
+#~ binutils
+#~ bison
+#~ fakeroot
+#~ file
+#~ findutils
+#~ flex
+#~ gawk
+#~ gcc
+#~ gcc
+#~ gettext
+#~ grep
+#~ groff
+#~ gzip
+#~ libtool
+#~ libtool
+#~ m4
+#~ make
+#~ pacman
+#~ patch
+#~ pkg-config
+#~ sed
+#~ sudo
+#~ systemd
+#~ texinfo
+#~ util-linux
+#~ which
+
+#~ stage2:
+#~ PACKAGES="bash
+#~ iana-etc filesystem linux-api-headers tzdata
+#~ ncurses readline joe
+#~ attr acl m4 gmp gdbm db perl openssl
+#~ libunistring gettext perl-locale-gettext help2man
+#~ autoconf automake perl-error pcre2 git libtool
+#~ zlib pambase cracklib libtirpc flex pam libcap coreutils
+#~ util-linux pkg-config e2fsprogs expat bzip2 lz4 xz pcre less gzip
+#~ tar libarchive curl
+#~ pacman-mirrorlist archlinux-keyring archlinux32-keyring pacman
+#~ elfutils sed texinfo grep findutils file diffutils ed patch
+#~ fakeroot
+#~ kbd procps-ng bison shadow
+#~ inetutils bc kmod linux uinit nasm 
+#~ net-tools libmnl libnfnetlink iptables iproute2
+#~ libedit openssh
+#~ make mpfr gawk libmpc binutils gcc glibc
+#~ libunwind strace gdb
+#~ "
+
+for p in $PACKAGES; do
+	"$SCRIPT_DIR/build_stage4_package.sh" "$p" || exit 1
+done
+
diff --git a/build_stage4_package.sh b/build_stage4_package.sh
new file mode 100755
index 0000000..de9d71b
--- /dev/null
+++ b/build_stage4_package.sh
@@ -0,0 +1,179 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# builds and installs one package for stage 3
+
+if test "$(id -u)" = 0; then
+	sudo -u cross "$0" "$1"
+	exit 0
+fi
+
+PACKAGE=$1
+
+# draw in default values for build variables
+
+. "$SCRIPT_DIR/$TARGET_CPU-stage4/template/DESCR"
+
+if test "$(find "$STAGE4_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then
+	echo "Building package $PACKAGE."
+
+	cd $STAGE4_BUILD || exit 1
+	
+	# clean up old build
+	
+	sudo rm -rf "$PACKAGE"
+	rm -f "$STAGE4_PACKAGES/$PACKAGE"-*pkg.tar.xz
+	ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP rm -rf "/build/$PACKAGE"
+
+	# check out the package build information from the upstream git rep
+	# using asp (or from the AUR using yaourt)
+	
+	PACKAGE_DIR="$SCRIPT_DIR/$TARGET_CPU-stage4/$PACKAGE"
+	PACKAGE_CONF="$PACKAGE_DIR/DESCR"	
+	if test -f "$PACKAGE_CONF"; then
+		if test "$(grep -c FETCH_METHOD "$PACKAGE_CONF")" = 1; then
+			FETCH_METHOD=$(grep FETCH_METHOD "$PACKAGE_CONF" | cut -f 2 -d = | tr -d '"')
+		fi
+	fi
+	case $FETCH_METHOD in
+		"asp")
+			$ASP export "$PACKAGE"
+			;;
+		"yaourt")
+			yaourt -G "$PACKAGE"
+			;;
+		"packages32")
+			# (we assume, we only take core packages)
+			cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" .
+			;;
+		*)
+			echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
+			exit 1
+	esac
+			
+	cd "$PACKAGE" || exit 1
+
+	# attach our destination platform to be a supported architecture
+	sed -i "/^arch=[^#]*any/!{/^arch=(/s/(/($TARGET_CPU /}" PKGBUILD
+
+	# if there is a packages32 diff-PKGBUILD, attach it at the end
+	# (we assume, we build only 'core' packages during stage1)
+	DIFF_PKGBUILD="$ARCHLINUX32_PACKAGES/core/$PACKAGE/PKGBUILD"
+	if test -f "$DIFF_PKGBUILD"; then
+		cat "$DIFF_PKGBUILD" >> PKGBUILD
+	fi
+
+	# copy all other files from Archlinux32, if they exist
+	# (we assume, we only take core packages during stage1)
+	if test -f "$DIFF_PKGBUILD"; then
+		find "$ARCHLINUX32_PACKAGES/core/$PACKAGE"/* ! -name PKGBUILD \
+			-exec cp {} . \;
+	fi
+		
+	# source package descriptions, sets variables for this script
+	# and executes whatever is needed to build the package
+
+	if test -f "$PACKAGE_CONF"; then
+		. "$PACKAGE_CONF"
+	fi
+
+	# copy all files into the build area on the target machine 
+	# (but the package DESCR file)
+	
+	if test -d "$PACKAGE_DIR"; then
+		find "$PACKAGE_DIR"/* ! -name DESCR \
+			-exec cp {} . \;
+	fi
+	
+	# execute makepkg on the host
+	# we would actually like to have a mode like 'download, and noextract' but
+	# makepkg is not doing that (see -e and -o options)
+	
+	makepkg --nobuild
+	rm -rf "$STAGE4_BUILD/$PACKAGE/src"
+
+	# copy everything to the stage 1 machine
+	
+	scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE4_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/.
+
+	# building the actual package
+
+	echo "Building $PACKAGE on target.."
+	
+	ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1
+	RES=$?
+	
+	tail "$PACKAGE.log"
+
+	if test $RES = 0; then
+	
+		echo "Package $PACKAGE built sucessfully, installing on target.."
+
+		# copy to our package folder in the first stage chroot
+		
+		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'
+			cd /build/$PACKAGE
+			rm -f ./*debug*.pkg.tar.xz
+			cp -v ./*.pkg.tar.xz /packages/$TARGET_CPU/.
+		'"
+
+		# redo the whole pacman cache and repo (always running into trouble
+		# there, packages seem to reappear in old versions)
+
+		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'
+#			rm -rf /var/cache/pacman/pkg/*
+#			rm -rf /packages/$TARGET_CPU/temp.db*
+#			rm -rf /packages/$TARGET_CPU/temp.files*
+#			repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz
+		'"
+		
+		# install onto stage 1 system via pacman
+
+		if test "$FORCE_INSTALL"; then
+			FORCE="--force"
+		fi
+		
+		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'		
+			# TODO: broken [temp] repo
+			if test \"$ADDITIONAL_INSTALL_PACKAGE\" != \"\"; then
+				#pacman $FORCE --noconfirm -Syy $PACKAGE $ADDITIONAL_INSTALL_PACKAGE
+				pacman $FORCE --noconfirm -U /packages/$TARGET_CPU/$PACKAGE-*.pkg.tar.xz /packages/$TARGET_CPU/$ADDITIONAL_INSTALL_PACKAGE-*.pkg.tar.xz
+			else
+				#pacman $FORCE --noconfirm -Syy $PACKAGE
+				pacman $FORCE --noconfirm -U /packages/$TARGET_CPU/$PACKAGE-*.pkg.tar.xz
+			fi
+		'"
+		
+		# copy packages from target machine and replace our local version with it
+
+		echo "Salvaging build environment of $PACKAGE from target back to host.."
+
+		tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir)
+		trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
+		
+		cd $STAGE4_BUILD || exit 1
+		mv "$STAGE4_BUILD/$PACKAGE/$PACKAGE.log" "$tmp_dir"
+		cd "$STAGE4_BUILD" || exit 1
+		rm -rf "$PACKAGE"
+		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && tar zcf $PACKAGE.tar.gz $PACKAGE/'"		
+		scp -i $CROSS_HOME/.ssh/id_rsa -rC build@$STAGE1_MACHINE_IP:/build/"$PACKAGE.tar.gz" "$STAGE4_BUILD/."
+		ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && rm -f $PACKAGE.tar.gz'"		
+		tar zxf "$PACKAGE.tar.gz"
+		rm -f "$PACKAGE.tar.gz"
+		mv "$tmp_dir/$PACKAGE.log" "$STAGE4_BUILD/$PACKAGE/."
+		mv -vf "$STAGE4_BUILD/$PACKAGE/"*.pkg.tar.xz "$STAGE4_PACKAGES/."
+
+		echo "Built package $PACKAGE."
+		
+	else	
+		echo "ERROR building package $PACKAGE"
+		exit 1
+	fi
+
+	cd $STAGE4_BUILD || exit 1
+
+else
+	echo "$PACKAGE exists."
+fi
diff --git a/default.conf b/default.conf
index ae4a9a1..54ed5e2 100644
--- a/default.conf
+++ b/default.conf
@@ -63,6 +63,15 @@ STAGE3_PACKAGES=$STAGE3_CHROOT/packages/$TARGET_CPU/
 # where build results are stored for stage 3
 STAGE3_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage3
 
+# the chroot of stage 4
+STAGE4_CHROOT=$CROSS_HOME/$TARGET_CPU-root-stage4
+
+# where packages are stored for stage 4 (on the host)
+STAGE4_PACKAGES=$STAGE4_CHROOT/packages/$TARGET_CPU/
+
+# where build results are stored for stage 4
+STAGE4_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage4
+
 # git repository for PKGBUILD diffs and patches for the
 # i686 architecture
 GIT_URL_ARCHLINUX32_PACKAGES=https://github.com/archlinux32/packages.git
diff --git a/i486-stage3/template/DESCR b/i486-stage3/template/DESCR
index 8a84b3a..80bb717 100644
--- a/i486-stage3/template/DESCR
+++ b/i486-stage3/template/DESCR
@@ -21,4 +21,4 @@ ADDITIONAL_INSTALL_PACKAGE=
 
 # FORCE_INSTALL = 0 | 1
 # per default packages should not be forced.
-FORCE_INSTALL=1
+FORCE_INSTALL=0
diff --git a/i486-stage4/template/DESCR b/i486-stage4/template/DESCR
new file mode 100644
index 0000000..80bb717
--- /dev/null
+++ b/i486-stage4/template/DESCR
@@ -0,0 +1,24 @@
+# FETCH_METHOD = "asp" | "yaourt" | "packages32"
+# "asp" is the default tool to fetch the package description PKGBUILD and
+# associated patch and other files.
+# If the package exists only on AUR, then use "yaourt".
+# If you want the package from Archlinux32, use "packages32".
+# (this is only for where to get the base set of files from, the patches
+# from packages32 are always applied)
+FETCH_METHOD="asp"
+
+# NOPARALLEL_BUILD = 0 | 1
+# the -j<N> parameter to makepkg will be set to -j if
+# NOPARALLEL_PACKAGE=1. The default is to use all avaiable cores
+# and set -j<CPUS>
+NOPARALLEL_BUILD=0
+
+# ADDITIONAL_INSTALL_PACKAGE = <package name>
+# per default the package has the same name as the package file,
+# some packages generate additional package files to install
+# (for example util-linux also has a libutil-linux)
+ADDITIONAL_INSTALL_PACKAGE=
+
+# FORCE_INSTALL = 0 | 1
+# per default packages should not be forced.
+FORCE_INSTALL=0
diff --git a/prepare_stage3.sh b/prepare_stage3_repo.sh
index 7b4fdcb..7b4fdcb 100755
--- a/prepare_stage3.sh
+++ b/prepare_stage3_repo.sh
diff --git a/prepare_stage4_repo.sh b/prepare_stage4_repo.sh
new file mode 100755
index 0000000..dbad677
--- /dev/null
+++ b/prepare_stage4_repo.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# prepare the i486-build for stage 4
+
+if test ! -d $STAGE4_CHROOT; then
+	mkdir $STAGE4_CHROOT
+	mkdir -p $STAGE4_PACKAGES
+fi
+
+if test ! -d $STAGE4_BUILD; then
+
+	# prepare the build enviroment
+	
+	mkdir $STAGE4_BUILD
+	cd $STAGE4_BUILD || exit 1
+
+	# TODO: actually build a STAGE4 hdd from the build artifacts of
+	# stage 3, for now we just copy the vm from stage2 after building
+	# and installing all packages from stage 3 and use it as new build
+	# machine.
+	
+	echo "Prepared the stage 4 build environment."
+fi
+
+echo "Stage 4 ready."