diff options
| -rw-r--r-- | README | 15 | ||||
| -rw-r--r-- | TODOS | 4 | ||||
| -rwxr-xr-x | build_stage1_package.sh | 2 | ||||
| -rwxr-xr-x | build_stage2_package.sh | 18 | ||||
| -rwxr-xr-x | build_stage3_package.sh | 10 | ||||
| -rwxr-xr-x | build_stage4.sh | 121 | ||||
| -rwxr-xr-x | build_stage4_package.sh | 179 | ||||
| -rw-r--r-- | default.conf | 9 | ||||
| -rw-r--r-- | i486-stage3/template/DESCR | 2 | ||||
| -rw-r--r-- | i486-stage4/template/DESCR | 24 | ||||
| -rwxr-xr-x | prepare_stage3_repo.sh (renamed from prepare_stage3.sh) | 0 | ||||
| -rwxr-xr-x | prepare_stage4_repo.sh | 28 |
12 files changed, 397 insertions, 15 deletions
@@ -219,6 +219,9 @@ su cross ./prepare_stage2_repo.sh # - libsecret, calls gtkdocize in autogen.sh # - systemd-sysvcompat: should be optional, I don't see why this should # be in base +# - nss: has big issues with hand-crafted assembler optimizations all +# over the place drawing in SSE2 as a requirement, used for +# building ca-certificates-mozilla only at the moment # systemd: goal is to have a libsystemd we can link against, but stage 3 # will not have a running systemd init system! @@ -226,7 +229,7 @@ su cross ./prepare_stage2_repo.sh su cross ./prepare_stage3_repo.sh # Build stage 3 on the target architecture and install it onto the -# stage 1 system. Resulting artifacts get stored also back +# stage 2 system. Resulting artifacts get stored also back # in $STAGE3_BUILD. # Most things compile just fine with 64 MB RAM, binutils ld gold needs more @@ -242,5 +245,15 @@ su cross ./prepare_stage3_repo.sh # use vanilla PKGBUILDs as much as possible, also building documentation # and do all the testing. +# We don't go to another build system for now till we can build the base +# system without patched PKGBUILDs (at least not where not needed). + # We also try to get a running systemd in this stage. +su cross ./prepare_stage4_repo.sh + +# Build stage 4 on the target architecture and install it onto the +# stage 3 system. Resulting artifacts get stored also back +# in $STAGE4_BUILD. + +./build_stage4.sh @@ -38,7 +38,9 @@ general bugs: works nevertheless - introduce a '-package' along to '(+)package' to delete a package at a given place (e.g. a shim) - +- get rid of ADDITIONAL_PACKAGES, we actually parse pkgname and install + all mentioned packages directly + stage1 issues: - stage1: cdrom installs keyrings without having a gpg binary - 32 MB is not enough when installing packages => add a swap as first action! diff --git a/build_stage1_package.sh b/build_stage1_package.sh index 97a81ad..dacd590 100755 --- a/build_stage1_package.sh +++ b/build_stage1_package.sh @@ -52,7 +52,7 @@ if test "$(pacman --config "$STAGE1_CHROOT/etc/pacman.conf" -r "$STAGE1_CHROOT" cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . ;; *) - print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 exit 1 esac diff --git a/build_stage2_package.sh b/build_stage2_package.sh index 2b58eff..ab06ba8 100755 --- a/build_stage2_package.sh +++ b/build_stage2_package.sh @@ -49,7 +49,7 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . ;; *) - print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 exit 1 esac @@ -99,6 +99,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE2_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/. # building the actual package + + echo "Building $PACKAGE on target.." ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1 RES=$? @@ -106,6 +108,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l tail "$PACKAGE.log" if test $RES = 0; then + + echo "Package $PACKAGE built sucessfully, installing on target.." # copy to our package folder in the first stage chroot @@ -119,10 +123,10 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l # there, packages seem to reappear in old versions) ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' - rm -rf /var/cache/pacman/pkg/* - rm -rf /packages/$TARGET_CPU/temp.db* - rm -rf /packages/$TARGET_CPU/temp.files* - repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz +# rm -rf /var/cache/pacman/pkg/* +# rm -rf /packages/$TARGET_CPU/temp.db* +# rm -rf /packages/$TARGET_CPU/temp.files* +# repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz '" # install onto stage 1 system via pacman @@ -130,7 +134,7 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l if test "$FORCE_INSTALL"; then FORCE="--force" fi - + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' # TODO: broken [temp] repo if test \"$ADDITIONAL_INSTALL_PACKAGE\" != \"\"; then @@ -144,6 +148,8 @@ if test "$(find "$STAGE2_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l # copy packages from target machine and replace our local version with it + echo "Salvaging build environment of $PACKAGE from target back to host.." + tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT diff --git a/build_stage3_package.sh b/build_stage3_package.sh index 2c2fdcb..c265212 100755 --- a/build_stage3_package.sh +++ b/build_stage3_package.sh @@ -16,7 +16,7 @@ PACKAGE=$1 . "$SCRIPT_DIR/$TARGET_CPU-stage3/template/DESCR" -#~ if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then +if test "$(find "$STAGE3_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then echo "Building package $PACKAGE." cd $STAGE3_BUILD || exit 1 @@ -49,7 +49,7 @@ PACKAGE=$1 cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . ;; *) - print "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 exit 1 esac @@ -174,6 +174,6 @@ PACKAGE=$1 cd $STAGE3_BUILD || exit 1 -#~ else - #~ echo "$PACKAGE exists." -#~ fi +else + echo "$PACKAGE exists." +fi diff --git a/build_stage4.sh b/build_stage4.sh new file mode 100755 index 0000000..40c1adb --- /dev/null +++ b/build_stage4.sh @@ -0,0 +1,121 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# build all packages for stage 4 using the target system with stage 3 +# packages. packages will be installed with pacman onto the target +# system once built sucessfully. The artifacts are also copied back +# to the $STAGE4_PACKAGES to speed up rebuild of the state of the stage 4 +# system in case of destroying it. + +PACKAGES="" + +# Archlinux base, base-devel groups +#~ bash +#~ bzip2 +#~ coreutils +#~ cryptsetup +#~ device-mapper +#~ dhcpcd +#~ diffutils +#~ e2fsprogs +#~ file +#~ filesystem +#~ findutils +#~ gawk +#~ gcc-libs +#~ gcc-libs +#~ gettext +#~ glibc +#~ grep +#~ gzip +#~ inetutils +#~ iproute2 +#~ iputils +#~ jfsutils +#~ less +#~ licenses +#~ linux +#~ logrotate +#~ lvm2 +#~ man-db +#~ man-pages +#~ mdadm +#~ nano +#~ netctl +#~ pacman +#~ pciutils +#~ pcmciautils +#~ perl +#~ procps-ng +#~ psmisc +#~ reiserfsprogs +#~ s-nail +#~ sed +#~ shadow +#~ sysfsutils +#~ systemd-sysvcompat +#~ tar +#~ texinfo +#~ usbutils +#~ util-linux +#~ vi +#~ which +#~ xfsprogs + +#~ autoconf +#~ automake +#~ binutils +#~ binutils +#~ bison +#~ fakeroot +#~ file +#~ findutils +#~ flex +#~ gawk +#~ gcc +#~ gcc +#~ gettext +#~ grep +#~ groff +#~ gzip +#~ libtool +#~ libtool +#~ m4 +#~ make +#~ pacman +#~ patch +#~ pkg-config +#~ sed +#~ sudo +#~ systemd +#~ texinfo +#~ util-linux +#~ which + +#~ stage2: +#~ PACKAGES="bash +#~ iana-etc filesystem linux-api-headers tzdata +#~ ncurses readline joe +#~ attr acl m4 gmp gdbm db perl openssl +#~ libunistring gettext perl-locale-gettext help2man +#~ autoconf automake perl-error pcre2 git libtool +#~ zlib pambase cracklib libtirpc flex pam libcap coreutils +#~ util-linux pkg-config e2fsprogs expat bzip2 lz4 xz pcre less gzip +#~ tar libarchive curl +#~ pacman-mirrorlist archlinux-keyring archlinux32-keyring pacman +#~ elfutils sed texinfo grep findutils file diffutils ed patch +#~ fakeroot +#~ kbd procps-ng bison shadow +#~ inetutils bc kmod linux uinit nasm +#~ net-tools libmnl libnfnetlink iptables iproute2 +#~ libedit openssh +#~ make mpfr gawk libmpc binutils gcc glibc +#~ libunwind strace gdb +#~ " + +for p in $PACKAGES; do + "$SCRIPT_DIR/build_stage4_package.sh" "$p" || exit 1 +done + diff --git a/build_stage4_package.sh b/build_stage4_package.sh new file mode 100755 index 0000000..de9d71b --- /dev/null +++ b/build_stage4_package.sh @@ -0,0 +1,179 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# builds and installs one package for stage 3 + +if test "$(id -u)" = 0; then + sudo -u cross "$0" "$1" + exit 0 +fi + +PACKAGE=$1 + +# draw in default values for build variables + +. "$SCRIPT_DIR/$TARGET_CPU-stage4/template/DESCR" + +if test "$(find "$STAGE4_PACKAGES" -regex ".*/$PACKAGE-.*pkg\\.tar\\.xz" | wc -l)" = 0; then + echo "Building package $PACKAGE." + + cd $STAGE4_BUILD || exit 1 + + # clean up old build + + sudo rm -rf "$PACKAGE" + rm -f "$STAGE4_PACKAGES/$PACKAGE"-*pkg.tar.xz + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP rm -rf "/build/$PACKAGE" + + # check out the package build information from the upstream git rep + # using asp (or from the AUR using yaourt) + + PACKAGE_DIR="$SCRIPT_DIR/$TARGET_CPU-stage4/$PACKAGE" + PACKAGE_CONF="$PACKAGE_DIR/DESCR" + if test -f "$PACKAGE_CONF"; then + if test "$(grep -c FETCH_METHOD "$PACKAGE_CONF")" = 1; then + FETCH_METHOD=$(grep FETCH_METHOD "$PACKAGE_CONF" | cut -f 2 -d = | tr -d '"') + fi + fi + case $FETCH_METHOD in + "asp") + $ASP export "$PACKAGE" + ;; + "yaourt") + yaourt -G "$PACKAGE" + ;; + "packages32") + # (we assume, we only take core packages) + cp -a "$ARCHLINUX32_PACKAGES/core/$PACKAGE" . + ;; + *) + echo "ERROR: unknown FETCH_METHOD '$FETCH_METHOD'.." >&2 + exit 1 + esac + + cd "$PACKAGE" || exit 1 + + # attach our destination platform to be a supported architecture + sed -i "/^arch=[^#]*any/!{/^arch=(/s/(/($TARGET_CPU /}" PKGBUILD + + # if there is a packages32 diff-PKGBUILD, attach it at the end + # (we assume, we build only 'core' packages during stage1) + DIFF_PKGBUILD="$ARCHLINUX32_PACKAGES/core/$PACKAGE/PKGBUILD" + if test -f "$DIFF_PKGBUILD"; then + cat "$DIFF_PKGBUILD" >> PKGBUILD + fi + + # copy all other files from Archlinux32, if they exist + # (we assume, we only take core packages during stage1) + if test -f "$DIFF_PKGBUILD"; then + find "$ARCHLINUX32_PACKAGES/core/$PACKAGE"/* ! -name PKGBUILD \ + -exec cp {} . \; + fi + + # source package descriptions, sets variables for this script + # and executes whatever is needed to build the package + + if test -f "$PACKAGE_CONF"; then + . "$PACKAGE_CONF" + fi + + # copy all files into the build area on the target machine + # (but the package DESCR file) + + if test -d "$PACKAGE_DIR"; then + find "$PACKAGE_DIR"/* ! -name DESCR \ + -exec cp {} . \; + fi + + # execute makepkg on the host + # we would actually like to have a mode like 'download, and noextract' but + # makepkg is not doing that (see -e and -o options) + + makepkg --nobuild + rm -rf "$STAGE4_BUILD/$PACKAGE/src" + + # copy everything to the stage 1 machine + + scp -i $CROSS_HOME/.ssh/id_rsa -rC "$STAGE4_BUILD/$PACKAGE" build@$STAGE1_MACHINE_IP:/build/. + + # building the actual package + + echo "Building $PACKAGE on target.." + + ssh -i $CROSS_HOME/.ssh/id_rsa build@$STAGE1_MACHINE_IP bash -c "'cd $PACKAGE && makepkg --skipchecksums --skippgpcheck --nocheck'" > $PACKAGE.log 2>&1 + RES=$? + + tail "$PACKAGE.log" + + if test $RES = 0; then + + echo "Package $PACKAGE built sucessfully, installing on target.." + + # copy to our package folder in the first stage chroot + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' + cd /build/$PACKAGE + rm -f ./*debug*.pkg.tar.xz + cp -v ./*.pkg.tar.xz /packages/$TARGET_CPU/. + '" + + # redo the whole pacman cache and repo (always running into trouble + # there, packages seem to reappear in old versions) + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' +# rm -rf /var/cache/pacman/pkg/* +# rm -rf /packages/$TARGET_CPU/temp.db* +# rm -rf /packages/$TARGET_CPU/temp.files* +# repo-add /packages/$TARGET_CPU/temp.db.tar.gz /packages/$TARGET_CPU/*pkg.tar.xz + '" + + # install onto stage 1 system via pacman + + if test "$FORCE_INSTALL"; then + FORCE="--force" + fi + + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "' + # TODO: broken [temp] repo + if test \"$ADDITIONAL_INSTALL_PACKAGE\" != \"\"; then + #pacman $FORCE --noconfirm -Syy $PACKAGE $ADDITIONAL_INSTALL_PACKAGE + pacman $FORCE --noconfirm -U /packages/$TARGET_CPU/$PACKAGE-*.pkg.tar.xz /packages/$TARGET_CPU/$ADDITIONAL_INSTALL_PACKAGE-*.pkg.tar.xz + else + #pacman $FORCE --noconfirm -Syy $PACKAGE + pacman $FORCE --noconfirm -U /packages/$TARGET_CPU/$PACKAGE-*.pkg.tar.xz + fi + '" + + # copy packages from target machine and replace our local version with it + + echo "Salvaging build environment of $PACKAGE from target back to host.." + + tmp_dir=$(mktemp -d 'tmp.compute-dependencies.0.XXXXXXXXXX' --tmpdir) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + + cd $STAGE4_BUILD || exit 1 + mv "$STAGE4_BUILD/$PACKAGE/$PACKAGE.log" "$tmp_dir" + cd "$STAGE4_BUILD" || exit 1 + rm -rf "$PACKAGE" + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && tar zcf $PACKAGE.tar.gz $PACKAGE/'" + scp -i $CROSS_HOME/.ssh/id_rsa -rC build@$STAGE1_MACHINE_IP:/build/"$PACKAGE.tar.gz" "$STAGE4_BUILD/." + ssh -i $CROSS_HOME/.ssh/id_rsa root@$STAGE1_MACHINE_IP bash -c "'cd /build && rm -f $PACKAGE.tar.gz'" + tar zxf "$PACKAGE.tar.gz" + rm -f "$PACKAGE.tar.gz" + mv "$tmp_dir/$PACKAGE.log" "$STAGE4_BUILD/$PACKAGE/." + mv -vf "$STAGE4_BUILD/$PACKAGE/"*.pkg.tar.xz "$STAGE4_PACKAGES/." + + echo "Built package $PACKAGE." + + else + echo "ERROR building package $PACKAGE" + exit 1 + fi + + cd $STAGE4_BUILD || exit 1 + +else + echo "$PACKAGE exists." +fi diff --git a/default.conf b/default.conf index ae4a9a1..54ed5e2 100644 --- a/default.conf +++ b/default.conf @@ -63,6 +63,15 @@ STAGE3_PACKAGES=$STAGE3_CHROOT/packages/$TARGET_CPU/ # where build results are stored for stage 3 STAGE3_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage3 +# the chroot of stage 4 +STAGE4_CHROOT=$CROSS_HOME/$TARGET_CPU-root-stage4 + +# where packages are stored for stage 4 (on the host) +STAGE4_PACKAGES=$STAGE4_CHROOT/packages/$TARGET_CPU/ + +# where build results are stored for stage 4 +STAGE4_BUILD=$CROSS_HOME/$TARGET_CPU-build-stage4 + # git repository for PKGBUILD diffs and patches for the # i686 architecture GIT_URL_ARCHLINUX32_PACKAGES=https://github.com/archlinux32/packages.git diff --git a/i486-stage3/template/DESCR b/i486-stage3/template/DESCR index 8a84b3a..80bb717 100644 --- a/i486-stage3/template/DESCR +++ b/i486-stage3/template/DESCR @@ -21,4 +21,4 @@ ADDITIONAL_INSTALL_PACKAGE= # FORCE_INSTALL = 0 | 1 # per default packages should not be forced. -FORCE_INSTALL=1 +FORCE_INSTALL=0 diff --git a/i486-stage4/template/DESCR b/i486-stage4/template/DESCR new file mode 100644 index 0000000..80bb717 --- /dev/null +++ b/i486-stage4/template/DESCR @@ -0,0 +1,24 @@ +# FETCH_METHOD = "asp" | "yaourt" | "packages32" +# "asp" is the default tool to fetch the package description PKGBUILD and +# associated patch and other files. +# If the package exists only on AUR, then use "yaourt". +# If you want the package from Archlinux32, use "packages32". +# (this is only for where to get the base set of files from, the patches +# from packages32 are always applied) +FETCH_METHOD="asp" + +# NOPARALLEL_BUILD = 0 | 1 +# the -j<N> parameter to makepkg will be set to -j if +# NOPARALLEL_PACKAGE=1. The default is to use all avaiable cores +# and set -j<CPUS> +NOPARALLEL_BUILD=0 + +# ADDITIONAL_INSTALL_PACKAGE = <package name> +# per default the package has the same name as the package file, +# some packages generate additional package files to install +# (for example util-linux also has a libutil-linux) +ADDITIONAL_INSTALL_PACKAGE= + +# FORCE_INSTALL = 0 | 1 +# per default packages should not be forced. +FORCE_INSTALL=0 diff --git a/prepare_stage3.sh b/prepare_stage3_repo.sh index 7b4fdcb..7b4fdcb 100755 --- a/prepare_stage3.sh +++ b/prepare_stage3_repo.sh diff --git a/prepare_stage4_repo.sh b/prepare_stage4_repo.sh new file mode 100755 index 0000000..dbad677 --- /dev/null +++ b/prepare_stage4_repo.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# shellcheck source=./default.conf +. "./default.conf" + +# prepare the i486-build for stage 4 + +if test ! -d $STAGE4_CHROOT; then + mkdir $STAGE4_CHROOT + mkdir -p $STAGE4_PACKAGES +fi + +if test ! -d $STAGE4_BUILD; then + + # prepare the build enviroment + + mkdir $STAGE4_BUILD + cd $STAGE4_BUILD || exit 1 + + # TODO: actually build a STAGE4 hdd from the build artifacts of + # stage 3, for now we just copy the vm from stage2 after building + # and installing all packages from stage 3 and use it as new build + # machine. + + echo "Prepared the stage 4 build environment." +fi + +echo "Stage 4 ready." |