building most unproblematic packages in stage3

13 files changed, 165 insertions, 64 deletions

diff --git a/README b/README
index b1c615e..19dbe1e 100644
--- a/README
+++ b/README
@@ -201,4 +201,12 @@ su cross ./prepare_stage2_repo.sh
 # dependencies). But we try to be as close to base + base-devel as possible
 # with minimal patching of PKGBUILDs.
 
+# Also ommit most documentation building as it has heavy dependencies.
+
 su cross ./prepare_stage3_repo.sh
+
+# Build stage 3 on the target architecture and install it onto the
+# stage 1 system. Resulting artifacts get stored also back
+# in $STAGE3_BUILD.
+
+./build_stage3.sh
diff --git a/TODOS b/TODOS
index 7484b87..6edae52 100644
--- a/TODOS
+++ b/TODOS
@@ -85,6 +85,5 @@ stage2 issues:
 
 stage 3 issues:
 - some packages don't install (also in stage 1): iana-etc, filesystem
-- withoud -x the messages of the build script are a little bit weird, add
-  some verbose output again
-  
+- can we build layers in parallel?
+
diff --git a/build_stage3.sh b/build_stage3.sh
index d1e60cd..3a3f15f 100755
--- a/build_stage3.sh
+++ b/build_stage3.sh
@@ -6,98 +6,111 @@
 # build all packages for stage 3 using the target system with stage 2
 # packages.  packages will be installed with pacman onto the target
 # system once built sucessfully. The artifacts are also copied back
-# to the $STAGE3_PACKAGES to speed up rebuild of the state of the stage 2
+# to the $STAGE3_PACKAGES to speed up rebuild of the state of the stage 3
 # system in case of destroying it.
 
 PACKAGES="iana-etc filesystem linux-api-headers tzdata
 ncurses readline bash joe
-attr acl"
+attr acl m4 gmp gdbm db perl openssl
+libunistring gettext perl-locale-gettext help2man
+autoconf automake perl-error pcre2 git libtool
+zlib pambase cracklib flex libcap
+e2fsprogs expat bzip2 lz4 xz pcre less gzip
+tar libarchive
+icu
+mpdecimal libffi 
+elfutils sed texinfo grep findutils file diffutils ed patch
+fakeroot
+bison
+inetutils bc kmod
+net-tools libmnl
+libedit
+argon2
+which
+"
 
 #~ stage2:
-#~  
-#~  m4 gmp gdbm db perl openssl
-#~ libunistring gettext perl-locale-gettext help2man
-#~ autoconf automake perl-error pcre2 git libtool
-#~ zlib pambase cracklib libtirpc flex pam libcap coreutils
-#~ util-linux pkg-config e2fsprogs expat bzip2 lz4 xz pcre less gzip
-#~ tar libarchive curl
-#~ pacman-mirrorlist archlinux-keyring archlinux32-keyring pacman
-#~ elfutils sed texinfo grep findutils file diffutils ed patch
-#~ fakeroot
-#~ kbd procps-ng bison shadow
-#~ inetutils bc kmod linux uinit nasm 
-#~ net-tools libmnl libnfnetlink iptables iproute2
-#~ libedit openssh
+#~ 
+# 
+# git problem, not base of base-devel, but needed to checkout out via
+# https+git protocoll. For now using the stage2 one
+#
+# libxml2: python2 and python as makedepends
+#~ libxslt: libxml2 libgcrypt python2
+#~ xmlto: libxslt docbook-xsl
+#~ git: python2 emacs libgnome-keyring xmlto asciidoc
+#~
+#~ libsasl: postgresql-libs libmariadbclient libldap krb5
+#~ libldap: libsasl
+#~ krb5: e2fsprogs libldap keyutils 
+#~ libtirpc: krb5
+
+#~ pam: w3m docbook-xml docbook-xsl
+#~         coreutils
+#~ util-linux:
+#~ pkg-config: glib2
+
+#~ libxml2: icu
+#~ libxslt: libxml2
+#~ libidn2 publicsuffix-list icu
+#~ libpsl: libidn2 libunistring libxslt python
+#~ curl: ca-certificates krb5 libssh2 libpsl libnghttp2
+
+#~ python: expat bzip2 gdbm openssl libffi zlib 
+
+#~ linux build full with mkinitcpio and modules
+
+#~ pacman-mirrorlist archlinux-keyring archlinux32-keyring pacman     
+#~ => kbd procps-ng <bison> shadow
+#~ <inetutils> <bc> <kmod> linux 
+#~ <net-tools> <libmnl> <libnfnetlink> iptables iproute2
+#~ <libedit> openssh
 #~ make mpfr gawk libmpc binutils gcc glibc
 #~ libunwind strace gdb
 #~ "
 #~ #TODO after nasm: syslinux
 
 # stage3 (from compute_dependencies.sh)
-#~ argon2: glibc 
-#~ autoconf: awk m4 diffutils sh 
-#~ automake: perl bash 
-#~ bash: readline glibc ncurses 
-#~ bc: readline 
 #~ binutils: glibc zlib 
-#~ bison: glibc m4 sh 
-#~ bzip2: glibc sh 
 #~ ca-certificates-cacert: ca-certificates-utils 
 #~ coreutils: glibc acl attr gmp libcap openssl 
 #~ cracklib: glibc zlib 
 #~ cryptsetup: device-mapper libgcrypt popt libutil-linux json-c argon2 
-#~ curl: ca-certificates krb5 libssh2 openssl zlib libpsl libnghttp2 
-#~ db: gcc-libs sh 
 #~ dbus: libsystemd expat 
 #~ dhcpcd: glibc sh udev libsystemd 
 #~ diffutils: glibc bash 
 #~ e2fsprogs: sh libutil-linux 
-#~ ed: glibc sh 
 #~ elfutils: gcc-libs zlib bzip2 xz 
-#~ expat: glibc 
 #~ fakeroot: glibc filesystem sed util-linux sh 
 #~ file: glibc zlib 
 #~ findutils: glibc sh 
 #~ flex: glibc m4 sh 
 #~ gawk: sh glibc mpfr 
 #~ gc: gcc-libs libatomic_ops 
-#~ gdbm: glibc sh 
-#~ gettext: gcc-libs acl sh glib2 libunistring 
 #~ glib2: pcre libffi libutil-linux zlib 
 #~ gmp: gcc-libs sh 
 #~ gnupg: npth libgpg-error libgcrypt libksba libassuan pinentry bzip2 readline gnutls sqlite 
 #~ gnutls: gcc-libs libtasn1 readline zlib nettle p11-kit libidn libunistring 
-#~ grep: glibc pcre 
 #~ groff: perl gcc-libs 
 #~ guile: gmp libltdl ncurses texinfo libunistring gc libffi 
 #~ guile2.0: gmp libltdl ncurses texinfo libunistring gc libffi 
-#~ gzip: glibc bash less 
-#~ inetutils: pam libcap 
-#~ iproute2: glibc iptables libelf 
 #~ iptables: glibc bash libnftnl libpcap 
 #~ iputils: openssl sysfsutils libcap libidn 
 #~ jfsutils: util-linux 
-#~ joe: ncurses 
 #~ json-c: glibc 
 #~ kbd: glibc pam 
 #~ keyutils: glibc sh 
-#~ kmod: glibc zlib xz 
-#~ krb5: e2fsprogs libldap keyutils 
 #~ ldns: openssl dnssec-anchors 
-#~ less: glibc ncurses pcre 
 #~ libarchive: acl attr bzip2 expat lz4 openssl xz zlib 
 #~ libassuan: libgpg-error 
 #~ libatomic_ops: glibc 
 #~ libcap: glibc attr 
 #~ libcap-ng: glibc 
 #~ libedit: ncurses 
-#~ libffi: glibc 
 #~ libgcrypt: libgpg-error 
 #~ libgpg-error: glibc sh 
 #~ libidn: glibc 
-#~ libidn2: libunistring 
 #~ libksba: bash libgpg-error glibc 
-#~ libmnl: glibc 
 #~ libmpc: mpfr 
 #~ libnfnetlink: glibc 
 #~ libnftnl: libmnl 
@@ -105,66 +118,70 @@ attr acl"
 #~ libnl: glibc 
 #~ libpcap: glibc libnl sh libusbx dbus 
 #~ libpipeline: glibc 
-#~ libpsl: libidn2 libunistring 
 #~ libseccomp: glibc 
 #~ libsecret: glib2 libgcrypt 
 #~ libssh2: openssl 
 #~ libtasn1: glibc 
 #~ libtirpc: krb5 
-#~ libtool: sh tar glibc 
-#~ libunistring: glibc 
 #~ libunwind: glibc xz 
 #~ libusb: glibc libsystemd 
 #~ logrotate: popt gzip acl 
-#~ lz4: glibc 
-#~ m4: glibc bash 
 #~ make: glibc guile 
 #~ man-db: bash gdbm zlib groff libpipeline less libseccomp 
 #~ mdadm: glibc 
 #~ mkinitcpio: awk mkinitcpio-busybox kmod util-linux libarchive coreutils bash findutils grep filesystem gzip systemd 
 #~ mpfr: gmp 
 #~ nano: ncurses file sh 
-#~ nasm: glibc 
-#~ ncurses: glibc gcc-libs 
-#~ net-tools: glibc 
 #~ netctl: coreutils iproute2 openresolv systemd 
 #~ nettle: gmp 
 #~ openresolv: bash 
 #~ openssh: krb5 openssl libedit ldns 
-#~ openssl: perl 
 #~ p11-kit: glibc libtasn1 libffi 
 #~ pacman: bash glibc libarchive curl gpgme pacman-mirrorlist archlinux-keyring 
 #~ pam: glibc cracklib libtirpc pambase 
-#~ patch: glibc attr 
 #~ pciutils: glibc hwids kmod 
 #~ pcmciautils: systemd 
-#~ pcre: gcc-libs readline zlib bzip2 bash 
-#~ pcre2: gcc-libs readline zlib bzip2 bash 
-#~ perl: gdbm db glibc 
 #~ pinentry: ncurses libcap libassuan libsecret 
 #~ pkg-config: glib2 
 #~ popt: glibc 
 #~ procps-ng: ncurses libsystemd 
 #~ psmisc: ncurses 
-#~ python: expat bzip2 gdbm openssl libffi zlib 
-#~ readline: glibc ncurses libncursesw.so 
 #~ reiserfsprogs: util-linux 
 #~ s-nail: openssl krb5 libidn 
-#~ sed: glibc acl attr 
 #~ shadow: bash pam acl 
 #~ strace: perl libunwind 
 #~ sudo: glibc libgcrypt pam libldap 
 #~ sysfsutils: glibc 
-#~ tar: glibc acl attr 
-#~ texinfo: ncurses gzip perl sh 
 #~ thin-provisioning-tools: expat gcc-libs libaio 
 #~ usbutils: libusb hwids 
 #~ vi: ncurses 
-#~ which: glibc bash 
 #~ xfsprogs: sh libutil-linux readline 
-#~ xz: sh 
-#~ zlib: glibc 
 
+#~ base cryptsetup
+#~ base device-mapper
+#~ base dhcpcd
+#~ base iputils
+#~ base jfsutils
+#~ base licenses
+#~ base logrotate
+#~ base lvm2
+#~ base man-db
+#~ base man-pages
+#~ base mdadm
+#~ base nano
+#~ base netctl
+#~ base pciutils
+#~ base pcmciautils
+#~ base psmisc
+#~ base reiserfsprogs
+#~ base s-nail
+#~ base systemd-sysvcompat
+#~ base usbutils
+#~ base vi
+#~ base xfsprogs
+#~ base-devel groff
+#~ base-devel sudo
+#~ base-devel systemd
 
 for p in $PACKAGES; do
 	"$SCRIPT_DIR/build_stage3_package.sh" "$p" || exit 1
diff --git a/build_stage3_package.sh b/build_stage3_package.sh
index 1c1242e..fe80592 100755
--- a/build_stage3_package.sh
+++ b/build_stage3_package.sh
@@ -99,6 +99,8 @@ if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 	scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE3_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/.
 
 	# building the actual package
+
+	echo "Building $PACKAGE on target.."
 	
 	ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1
 	RES=$?
@@ -106,6 +108,8 @@ if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 	tail "$PACKAGE.log"
 
 	if test $RES = 0; then
+	
+		echo "Package $PACKAGE built sucessfully, installing on target.."
 
 		# copy to our package folder in the first stage chroot
 		
@@ -139,6 +143,8 @@ if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l
 		
 		# copy packages from target machine and replace our local version with it
 
+		echo "Salvaging build environment of $PACKAGE from target back to host.."
+
 		tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir)
 		trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
 		
diff --git a/i486-stage3/fakeroot/DESCR b/i486-stage3/fakeroot/DESCR
new file mode 100644
index 0000000..ef8246d
--- /dev/null
+++ b/i486-stage3/fakeroot/DESCR
@@ -0,0 +1,4 @@
+# disable makedepends and take out po4a
+sed -i "/makedepends=/s/po4a//" PKGBUILD
+sed -i 's@\(cd doc\)@#\1@g' PKGBUILD
+sed -i 's@\(po4a\)@#\1@g' PKGBUILD
diff --git a/i486-stage3/gettext/DESCR b/i486-stage3/gettext/DESCR
new file mode 100644
index 0000000..a6a9ab3
--- /dev/null
+++ b/i486-stage3/gettext/DESCR
@@ -0,0 +1,4 @@
+# remove dependency on glib2 (which needs meson to build) and use
+# supplied one
+sed -i "/depends=/s/'glib2'//" PKGBUILD
+sed -i "s@./configure@./configure --with-included-glib@" PKGBUILD
diff --git a/i486-stage3/git/DESCR b/i486-stage3/git/DESCR
new file mode 100644
index 0000000..e95d538
--- /dev/null
+++ b/i486-stage3/git/DESCR
@@ -0,0 +1,23 @@
+# no python2
+sed -i "/makedepends=/s/'python2'//" PKGBUILD
+
+# no Gnome integration
+sed -i "/makedepends=/s/'libgnome-keyring'//" PKGBUILD
+sed -i "s@\(make -C contrib/credential/gnome-keyring\)@#\1@" PKGBUILD
+
+# no Emacs
+sed -i "/makedepends=/s/'emacs'//" PKGBUILD
+sed -i "s@\(make -C contrib/emacs prefix=/usr\)@#\1@" PKGBUILD
+
+# no documentation
+sed -i "/makedepends=/s/'xmlto'//" PKGBUILD
+sed -i "/makedepends=/s/'asciidoc'//" PKGBUILD
+sed -i "s/all doc/all/g" PKGBUILD
+sed -i "s/install install-doc/install/g" PKGBUILD
+
+# git-credential-libsecret.c:29:10: fatal error: glib.h: No such file or directory
+# include <glib.h>
+sed -i "s@\(make -C contrib/credential/libsecret\)@#\1@" PKGBUILD
+sed -i "/# gnome credentials helper/,/make -C contrib\/credential\/libsecret clean/s/\(.*\)/#\1/g" PKGBUILD
+
+# not configured with configure? weird..
diff --git a/i486-stage3/kmod/DESCR b/i486-stage3/kmod/DESCR
new file mode 100644
index 0000000..8a94b92
--- /dev/null
+++ b/i486-stage3/kmod/DESCR
@@ -0,0 +1,3 @@
+# don't build documentation
+sed -i "/depends=/s/gtk-doc//" PKGBUILD
+sed -i "s/--enable-gtk-doc/--disable-gtk-doc/" PKGBUILD
diff --git a/i486-stage3/libidn2/DESCR b/i486-stage3/libidn2/DESCR
new file mode 100644
index 0000000..e4157fa
--- /dev/null
+++ b/i486-stage3/libidn2/DESCR
@@ -0,0 +1,3 @@
+# remove dependency on gtk-doc and ruby-ronn 
+sed -i "/makedepends=/s/'gtk-doc'//" PKGBUILD
+sed -i "/makedepends=/s/'ruby-ronn'//" PKGBUILD
diff --git a/i486-stage3/libpsl/DESCR b/i486-stage3/libpsl/DESCR
new file mode 100644
index 0000000..e80b78f
--- /dev/null
+++ b/i486-stage3/libpsl/DESCR
@@ -0,0 +1,11 @@
+# remove dependency python
+sed -i "/makedepends=/s/'python'//" PKGBUILD
+
+# no documentation
+sed -i "/makedepends=/s/'gtk-doc'//" PKGBUILD
+sed -i "s@--enable-gtk-doc@--disable-gtk-doc@" PKGBUILD
+
+# no man pages
+sed -i "/makedepends=/s/'libxslt'//" PKGBUILD
+sed -i "s@--enable-man@--disable-man@" PKGBUILD
+
diff --git a/i486-stage3/libxml2/DESCR b/i486-stage3/libxml2/DESCR
new file mode 100644
index 0000000..d90eea6
--- /dev/null
+++ b/i486-stage3/libxml2/DESCR
@@ -0,0 +1,2 @@
+# remove dependency python
+sed -i "/makedepends=/s/'python'//" PKGBUILD
diff --git a/i486-stage3/openssl/DESCR b/i486-stage3/openssl/DESCR
new file mode 100644
index 0000000..69d65f7
--- /dev/null
+++ b/i486-stage3/openssl/DESCR
@@ -0,0 +1,5 @@
+sed -i "s@'i686'@'$TARGET_CPU'@g" PKGBUILD
+sed -i "s@'linux-elf'@'linux-generic32'@g" PKGBUILD
+sed -i 's@shared no-ssl3-method ${optflags}@shared ${optflags}@' PKGBUILD
+
+# pod2man fails during building of documentation, not dramatic, ignoring that.
diff --git a/i486-stage3/python/DESCR b/i486-stage3/python/DESCR
new file mode 100644
index 0000000..1e44107
--- /dev/null
+++ b/i486-stage3/python/DESCR
@@ -0,0 +1,16 @@
+# remove dependency on most things, we want only a simple interpreter
+# a little bit smarter than a shell for building and code generation
+sed -i "/makedepends=/s/'tk'//" PKGBUILD
+sed -i "/makedepends=/s/'sqlite'//" PKGBUILD
+sed -i "/makedepends=/s/'valgrind'//" PKGBUILD
+sed -i "/makedepends=/s/'bluez-libs'//" PKGBUILD
+sed -i "/makedepends=/s/'llvm'//" PKGBUILD
+
+# do not start X servers just for building software!
+sed -i "/makedepends=/s/'xorg-server-xvfb'//" PKGBUILD
+sed -i "s/\(export servernum=99\)/#\1/" PKGBUILD
+sed -i "s/\(while\)/#\1/" PKGBUILD
+sed -i 's@LC_CTYPE=en_US.UTF-8 xvfb-run -a -n "$servernum" make EXTRA_CFLAGS="$CFLAGS"@LC_CTYPE=en_US.UTF-8 make EXTRA_CFLAGS="$CFLAGS"@g' PKGBUILD
+
+# no LTO (we don't have support for it yet)
+sed -i "s/--with-lto/--without-lto/" PKGBUILD