work on stage 2

9 files changed, 242 insertions, 13 deletions

diff --git a/README b/README
index f17ece6..9af3759 100644
--- a/README
+++ b/README
@@ -163,3 +163,26 @@ su cross ./create_cdrom.sh
 # stage 2 build system on the target architecture)
 
 su cross ./create_hdd.sh
+
+#########
+# STAGE 2
+#########
+
+# Assume stage1 of the system is installed on the target architecture
+# (virtual or a real machine), we have a key-based SSH connection to
+# the machine and can build packages there.
+
+# Build stage2 in $STAGE2_BUILD with the tools on the stage1 system
+# and modified PKGBUILDs and patches into the target system (replacing
+# stage 1 packages). Stage 1 serves now the same function as the sysroot.
+
+# The goal is to get a self-hosting system on the target architecture,
+# not necesseraly containing all of base and base-devel, but enough to
+# bootstrap itself on the target architecture.
+
+su cross ./prepare_stage2_repo.sh
+
+# Build stage 2 on the target architecture and install it onto the
+# stage 1 system. Resulting artifacts get stored in $STAGE2_BUILD.
+
+./build_stage2.sh
diff --git a/TODOS b/TODOS
index 56aa715..e1cb7c8 100644
--- a/TODOS
+++ b/TODOS
@@ -1,3 +1,5 @@
+general bugs:
+
 - how to clean up sysroot nicely without having to rebuild the cross-compiler?
   for now only removing and stowing a copy somewhere before building stage1
   helps.
@@ -12,8 +14,6 @@
   bsdtar: /home/cross/i486-root/packages/i486/linux-headers-4.15.1-2-i486.pkg.tar.xz: Not found in archive
   bsdtar: Error exit delayed from previous errors.
   Built package linux.
-- currently gcc has a PKGBUILD modified by hand instead of diff seds in DESCR
-  (the diff is very big and complex)
 - removing all pacman databases and recreating them with repo-add, also
   removing all installed pacman packages seem a little bit paranoic. Find
   out how to speed this up and do it properly.
@@ -29,19 +29,12 @@
   error: no usable package repositories configured.
   installing and listing packages by hand works without problems.
   debug: unregistering database 'local'
-- make: recursive tarkets like all-recursive don't work, neither with cross-compiled
-  make nor with the recompiled make on the stage1 system.
-  => rebuilding bash on the target works, but it's a workaround:
-  https://unix.stackexchange.com/questions/389022/make-unable-to-recurse-for-autoconf-like-projects/423448#423448
 - some packages get build more than once in stage1 (linux-api-headers, pacman-mirrorlist)
 - make a wrapper for sed in DESCR, which is actually able to detect whether
   the patch was successful or not. Avoid hard to detect errors, when the
   upstream PKGBUILD(s) change.
-- gdb uses cross-compiler source files for debugging:
-  Temporary breakpoint 1, 0x004005b0 in main ()
-  (gdb) list
-  1       /home/cross/.build/i486-unknown-linux-gnu/src/gcc/libgcc/libgcc2.c: No such file or directory.
-- stage1
+
+stage1 issues:
   - stage1: cdrom installs keyrings without having a gpg binary
   - 32 MB is not enough when installing packages => add a swap as first action!
   - stage1 has no bootloader package (syslinux) to install
@@ -49,3 +42,17 @@
 - using the cross compiler for syslinux is most likely overkill
 - some packages still build more than really needed:
   - syslinux: builds docu with asciidoc and build EFI stuff from gnu-efi
+- rename STAGE1 directories i486-root to i486-stage1-root and i486-build
+  to i486-build-stage1
+- gdb uses cross-compiler source files for debugging:
+  Temporary breakpoint 1, 0x004005b0 in main ()
+  (gdb) list
+  1       /home/cross/.build/i486-unknown-linux-gnu/src/gcc/libgcc/libgcc2.c: No such file or directory.
+- currently gcc has a PKGBUILD modified by hand instead of diff seds in DESCR
+  (the diff is very big and complex)
+
+stage2 issues:  
+- make: recursive tarkets like all-recursive don't work, neither with cross-compiled
+  make nor with the recompiled make on the stage1 system.
+  => rebuilding bash on the target works, but it's a workaround:
+  https://unix.stackexchange.com/questions/389022/make-unable-to-recurse-for-autoconf-like-projects/423448#423448
diff --git a/build_stage2.sh b/build_stage2.sh
new file mode 100755
index 0000000..8cf5dfd
--- /dev/null
+++ b/build_stage2.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# build all packages for stage 2 using the target system with stage 1
+# packages.  packages will be installed with pacman onto the target
+# system once built sucessfully. The artifacts are also copied back
+# to the $STAGE2_PACKAGES to speed up rebuild of the state of the stage 1
+# system in case of destroying it.
+
+PACKAGES="bash"
+
+for p in $PACKAGES; do
+	"$SCRIPT_DIR/build_stage2_package.sh" "$p" || exit 1
+done
+
diff --git a/build_stage2_package.sh b/build_stage2_package.sh
new file mode 100755
index 0000000..ebf75da
--- /dev/null
+++ b/build_stage2_package.sh
@@ -0,0 +1,101 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# builds and installs one package for stage 1
+
+if test "$(id -u)" = 0; then
+	sudo -u cross "$0" "$1"
+	exit 0
+fi
+
+PACKAGE=$1
+
+# draw in default values for build variables
+
+. "$SCRIPT_DIR/$TARGET_CPU-stage2/template/DESCR"
+
+if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz"n | wc -l)" = 0; then
+	echo "Building package $PACKAGE."
+
+	cd $STAGE2_BUILD || exit 1
+	
+	# clean up old build
+	
+	sudo rm -rf "$PACKAGE"
+	rm -f "$STAGE2_PACKAGES/$PACKAGE"-*pkg.tar.xz
+	ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP rm -rf "/build/$PACKAGE"
+
+	# check out the package build information from the upstream git rep
+	# using asp (or from the AUR using yaourt)
+	
+	PACKAGE_DIR="$SCRIPT_DIR/$TARGET_CPU-stage2/$PACKAGE"
+	PACKAGE_CONF="$PACKAGE_DIR/DESCR"	
+	if test -f "$PACKAGE_CONF"; then
+		if test "$(grep -c FETCH_METHOD "$PACKAGE_CONF")" = 1; then
+			FETCH_METHOD=$(grep FETCH_METHOD "$PACKAGE_CONF" | cut -f 2 -d = | tr -d '"')
+		fi
+	fi
+	case $FETCH_METHOD in
+		"asp")
+			$ASP export "$PACKAGE"
+			;;
+		"yaourt")
+			yaourt -G "$PACKAGE"
+			;;
+		"packages32")
+			# (we assume, we only take core packages)
+			cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" .
+			;;
+		*)
+			print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2
+			exit 1
+	esac
+			
+	cd "$PACKAGE" || exit 1
+
+	# attach our destination platform to be a supported architecture
+	sed -i "/^arch=[^#]*any/!{/^arch=(/s/(/($TARGET_CPU /}" PKGBUILD
+
+	# if there is a packages32 diff-PKGBUILD, attach it at the end
+	# (we assume, we build only 'core' packages during stage1)
+	DIFF_PKGBUILD="$ARCHLINUX32_PACKAGES/core/$PACKAGE/PKGBUILD"
+	if test -f "$DIFF_PKGBUILD"; then
+		cat "$DIFF_PKGBUILD" >> PKGBUILD
+	fi
+
+	# copy all other files from Archlinux32, if they exist
+	# (we assume, we only take core packages during stage1)
+	if test -f "$DIFF_PKGBUILD"; then
+		find "$ARCHLINUX32_PACKAGES/core/$PACKAGE"/* ! -name PKGBUILD \
+			-exec cp {} . \;
+	fi
+		
+	# source package descriptions, sets variables for this script
+	# and executes whatever is needed to build the package
+
+	if test -f "$PACKAGE_CONF"; then
+		. "$PACKAGE_CONF"
+	fi
+
+	# copy all files into the build area on the target machine 
+	# (but the package DESCR file)
+	if test -d "$PACKAGE_DIR"; then
+		find "$PACKAGE_DIR"/* ! -name DESCR \
+			-exec cp {} . \;
+	fi
+	
+	# execute makepkg on the host, we don't have git on the stage 1 machine (yet)
+	makepkg --nobuild
+
+	# copy everything to the stage 1 machine
+	scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE2_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/.
+
+	# TODO:
+	# building the actual package
+	#$STAGE1_BUILD/makepkg-$TARGET_CPU -C --config $STAGE1_BUILD/makepkg-$TARGET_CPU.conf \
+#		--skipchecksums --skippgpcheck --nocheck > "$PACKAGE.log" 2>&1
+#	RES=$?
+
+fi
diff --git a/create_hdd.sh b/create_hdd.sh
index 3170a64..94df476 100755
--- a/create_hdd.sh
+++ b/create_hdd.sh
@@ -53,7 +53,7 @@ mount -t devpts devpts /dev/pts
 mount -t sysfs sys /sys
 mount -o remount,rw /
 ip link set up dev eth0
-ip addr add 192.168.1.127/24 dev eth0
+ip addr add ${STAGE1_MACHINE_IP}/24 dev eth0
 ip route add default via 192.168.1.1 dev eth0
 /usr/sbin/sshd
 exec /usr/bin/bash
@@ -81,6 +81,21 @@ chmod 0400 etc/ssh/ssh_host_*_key
 mkdir root/.ssh
 cp "$HOME/.ssh/id_rsa.pub" root/.ssh/authorized_keys
 
+# install a build user and build directory
+cat >> etc/group <<EOF
+build:x:1001:
+EOF
+cat >> etc/passwd <<EOF
+build:x:1001:1001:build:/build:/bin/bash
+EOF
+mkdir -p build
+mkdir build/.ssh
+cp "$HOME/.ssh/id_rsa.pub" build/.ssh/authorized_keys
+chown 1001:1001 build
+# TODO: why does su require a password though we want to login via
+# SSH and key only!?
+#echo 'build:xx' | chpasswd
+
 # add some test programs to test the C and C++ compiler
 
 cat > root/test.c <<EOF
@@ -106,6 +121,9 @@ int main( void )
 }
 EOF
 
+# put proper pacman.conf in place
+mv etc/pacman.conf.pacnew etc/pacman.conf
+
 # fix permissions (we only have root on the image)
 
 chmod 0700 root/.ssh
diff --git a/default.conf b/default.conf
index e28de34..a8b82dc 100644
--- a/default.conf
+++ b/default.conf
@@ -25,17 +25,33 @@ XTOOLS_ARCH_PREFIX=$XTOOLS_ARCH/bin/$TARGET_ARCH-
 KERNEL_ARCH=i386
 
 # the chroot of stage 1
+# TODO: rename
 STAGE1_CHROOT=$CROSS_HOME/$TARGET_CPU-root
 
 # the place where we build stage 1
+# TODO: rename
 STAGE1_BUILD=$CROSS_HOME/$TARGET_CPU-build
 
 # where packages are stored for stage 1
+# TODO: rename
 STAGE1_PACKAGES=$STAGE1_CHROOT/packages/$TARGET_CPU/
 
 # where to store the directory which go to the ISO image
+# TODO: rename
 STAGE1_ISOLINUX=$CROSS_HOME/isolinux
 
+# where is stage1 installed on the target architecture and reachable
+STAGE1_MACHINE_IP=192.168.1.127
+
+# where packages are stored for stage 2
+STAGE2_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage2
+
+# the chroot of stage 2
+STAGE2_CHROOT=$CROSS_HOME/$TARGET_CPU-root-stage2
+
+# where packages are stored for stage 2 (on the host)
+STAGE2_PACKAGES=$STAGE2_CHROOT/packages/$TARGET_CPU/
+
 # git repository for PKGBUILD diffs and patches for the
 # i686 architecture
 GIT_URL_ARCHLINUX32_PACKAGES=https://github.com/archlinux32/packages.git
@@ -44,7 +60,7 @@ GIT_URL_ARCHLINUX32_PACKAGES=https://github.com/archlinux32/packages.git
 ARCHLINUX32_PACKAGES=$CROSS_HOME/packages32
 
 # uncomment to debug scripts
-#set -x
+set -x
 
 # some default variables, not to be changed
 SCRIPT=$(sudo realpath -s "$0")
diff --git a/i486-stage2/bash/DESCR b/i486-stage2/bash/DESCR
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/i486-stage2/bash/DESCR
diff --git a/i486-stage2/template/DESCR b/i486-stage2/template/DESCR
new file mode 100644
index 0000000..cb3d05e
--- /dev/null
+++ b/i486-stage2/template/DESCR
@@ -0,0 +1,20 @@
+# FETCH_METHOD = "asp" | "yaourt" | "packages32"
+# "asp" is the default tool to fetch the package description PKGBUILD and
+# associated patch and other files.
+# If the package exists only on AUR, then use "yaourt".
+# If you want the package from Archlinux32, use "packages32".
+# (this is only for where to get the base set of files from, the patches
+# from packages32 are always applied)
+FETCH_METHOD="asp"
+
+# NOPARALLEL_BUILD = 0 | 1
+# the -j<N> parameter to makepkg will be set to -j if
+# NOPARALLEL_PACKAGE=1. The default is to use all avaiable cores
+# and set -j<CPUS>
+NOPARALLEL_BUILD=0
+
+# ADDITIONAL_INSTALL_PACKAGE = <package name>
+# per default the package has the same name as the package file,
+# some packages generate additional package files to install
+# (for example util-linux also has a libutil-linux)
+ADDITIONAL_INSTALL_PACKAGE=
diff --git a/prepare_stage2_repo.sh b/prepare_stage2_repo.sh
new file mode 100755
index 0000000..5ce5f00
--- /dev/null
+++ b/prepare_stage2_repo.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# shellcheck source=./default.conf
+. "./default.conf"
+
+# prepare the i486-build for stage 2
+
+if test ! -d $STAGE2_CHROOT; then
+	mkdir $STAGE2_CHROOT
+	mkdir -p $STAGE2_PACKAGES
+fi
+
+if test ! -d $STAGE2_BUILD; then
+
+	# prepare the build enviroment
+	
+	mkdir $STAGE2_BUILD
+	cd $STAGE2_BUILD || exit 1
+
+	# TODO: check makepkg in stage1 hdd
+	# TODO: makepkg patch to run as root or add a build user to hdd
+	# TODO: we assume we have on cpu for now on the target
+	
+	echo "Prepared the stage 2 build environment."
+fi
+
+echo "Stage 2 ready."