From 9a97b6c83f5f73840e6cfb0bf55681b7a350b778 Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 1 Oct 2023 19:29:48 +0200 Subject: [PATCH 1/8] =?UTF-8?q?Add=20details=20about=20the=20image?= =?UTF-8?q?=E2=80=99s=20architecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build-image.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build-image.sh b/build-image.sh index 4f2e489..7eb73fe 100755 --- a/build-image.sh +++ b/build-image.sh @@ -16,6 +16,10 @@ BINPKG="${BINPKG:-/var/cache/packages-podman-1}" LOGDIR="${LOGDIR:-$(pwd)/log}" DOCKER_FILE="${DOCKER_FILE:-$(pwd)/Dockerfile}" +ARCH="${ARCH:-amd64}" +MICROARCH="${MICROARCH:-amd64}" +OCI_ARCH="${OCI_ARCH:-linux/amd64}" + PODMAN_BUILD_ARGS=( # Do not leak the host's /etc/host into the container. --no-hosts @@ -34,6 +38,10 @@ PODMAN_BUILD_ARGS=( -v "${LOGDIR}:/var/log:rw,U" # Use the given OCI file/Dockerfile. -f "${DOCKER_FILE}" + # Add details about the architecture. + --build-arg ARCH="${ARCH}" + --build-arg MICROARCH="${MICROARCH}" + --platform "${OCI_ARCH}" # Tag the generated image. -t "${IMAGE_TAG}" # Label the image. From 1eee4143609b361eabfe4781ed6612be48151b70 Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 1 Oct 2023 19:34:43 +0200 Subject: [PATCH 2/8] Add more comments --- build-image.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build-image.sh b/build-image.sh index 7eb73fe..11c1140 100755 --- a/build-image.sh +++ b/build-image.sh @@ -63,17 +63,23 @@ _mkdir() mkdir -p "${@}" || exit_err "Could not create dir ${@}." } +# === Prepare all directories. _mkdir "${REPOS}" _mkdir "${DISTFILES}" _mkdir "${BINPKG}" _mkdir "${LOGDIR}" +# === Fetch the base image. podman pull gentoo/stage3:amd64-nomultilib-systemd || exit_err "Could not fetch the image." + +# === Build the new image. podman build "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." -# Update the tag 'latest'. +# === Remove the old tag 'latest'. podman tag rm "${REGISTRY}:latest" # Do not exit_err here. At least on first run # there is no latest tag to delete. + +# === Tage the new image as 'latest'. podman tag "${REGISTRY}:${VERSION}" "${REGISTRY}:latest" || exit_err "Could not tag new image as 'latest'." # vim:fileencoding=utf-8:ts=4:syntax=bash:expandtab From 0ca32aa2ae2ee32c21fa4f0c013dc0056e188da7 Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 1 Oct 2023 19:35:23 +0200 Subject: [PATCH 3/8] Re-order tagging and tag cleanup --- build-image.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/build-image.sh b/build-image.sh index 11c1140..3556920 100755 --- a/build-image.sh +++ b/build-image.sh @@ -44,6 +44,7 @@ PODMAN_BUILD_ARGS=( --platform "${OCI_ARCH}" # Tag the generated image. -t "${IMAGE_TAG}" + -t "latest" # Label the image. --label="gentoo-nfr-${IMAGE_TAG}" # Sign the image. @@ -72,14 +73,11 @@ _mkdir "${LOGDIR}" # === Fetch the base image. podman pull gentoo/stage3:amd64-nomultilib-systemd || exit_err "Could not fetch the image." -# === Build the new image. -podman build "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." - # === Remove the old tag 'latest'. podman tag rm "${REGISTRY}:latest" # Do not exit_err here. At least on first run # there is no latest tag to delete. -# === Tage the new image as 'latest'. -podman tag "${REGISTRY}:${VERSION}" "${REGISTRY}:latest" || exit_err "Could not tag new image as 'latest'." +# === Build the new image. +podman build "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." # vim:fileencoding=utf-8:ts=4:syntax=bash:expandtab From 8642162cb00d25b4e911e65cd2d3af699f38a491 Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 1 Oct 2023 21:10:07 +0200 Subject: [PATCH 4/8] Import stage3 directly --- Dockerfile | 9 +++++---- build-image.sh | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 790e671..001920c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,11 @@ # =========================================================================== # Update the source image. # =========================================================================== -FROM gentoo/stage3:amd64-nomultilib-systemd as bootstrap -# Migrate to a merged-usr form. -RUN emerge --quiet-build=y --oneshot merge-usr -RUN merge-usr + +FROM scratch as bootstrap +# Unpack the stage3 archive (which is downloaded and verifed externally). +ADD latest-stage3.tar.xz / +# Set the profile. RUN eselect profile set "default/linux/amd64/17.1/no-multilib/systemd/merged-usr" # Replace /etc/portage/make.conf. RUN rm --one-file-system /etc/portage/make.conf diff --git a/build-image.sh b/build-image.sh index 3556920..22ce9f0 100755 --- a/build-image.sh +++ b/build-image.sh @@ -47,7 +47,7 @@ PODMAN_BUILD_ARGS=( -t "latest" # Label the image. --label="gentoo-nfr-${IMAGE_TAG}" - # Sign the image. + # sign the image. #--sign-by="${GPG_SIGNING_KEY}" # Rebuild everything w/o cache. --no-cache @@ -70,8 +70,21 @@ _mkdir "${DISTFILES}" _mkdir "${BINPKG}" _mkdir "${LOGDIR}" -# === Fetch the base image. -podman pull gentoo/stage3:amd64-nomultilib-systemd || exit_err "Could not fetch the image." +# === Fetch the stage3 file (and verify it). +SERVER="https://ftp-osl.osuosl.org/pub/gentoo/releases/${ARCH}/autobuilds" +MY_STAGE3="latest-stage3-amd64-nomultilib-systemd-mergedusr.txt" +LATEST_ARCHIVE="$(curl -sLC- ${SERVER}/${MY_STAGE3} | tail -n1 | cut -f1 -d' ')" +ARCHIVE_FILE_NAME="$(echo ${LATEST_ARCHIVE} | cut -f2 -d'/')" + +### Fetch the stage3 archive and its signature. +curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}" \ + || exit_err "Could not download the stage3 archive." +curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}.asc" \ + || exit_err "Could not download the stage3 archive signature." + +# Verify the signature. +gpg --verify "${DISTFILES}/${ARCHIVE_FILE_NAME}"{.asc,} \ + || exit_err "Could not verify the stage3 archive." # === Remove the old tag 'latest'. podman tag rm "${REGISTRY}:latest" # Do not exit_err here. At least on first run From 86f6d4e8c437032e83fdae7beb530be900fb8c3a Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Wed, 18 Oct 2023 20:39:06 +0200 Subject: [PATCH 5/8] Fix latest tag --- build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-image.sh b/build-image.sh index 22ce9f0..7d8d636 100755 --- a/build-image.sh +++ b/build-image.sh @@ -44,7 +44,7 @@ PODMAN_BUILD_ARGS=( --platform "${OCI_ARCH}" # Tag the generated image. -t "${IMAGE_TAG}" - -t "latest" + -t "${REGISTRY}:latest" # Label the image. --label="gentoo-nfr-${IMAGE_TAG}" # sign the image. From accf8c33e7c6b71907a3eb6cf360d8304832654e Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 22 Oct 2023 11:17:17 +0200 Subject: [PATCH 6/8] Provide the stage3 file via ARGS My distfiles path on my host does not match the one mounted into the container. Symlinks are not followed by podman, AFAICT due to security concerns. Using ARGS to change the path seems reasonable to me. --- Dockerfile | 3 ++- build-image.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 001920c..4569f50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ FROM scratch as bootstrap # Unpack the stage3 archive (which is downloaded and verifed externally). -ADD latest-stage3.tar.xz / +ARG ROOTFS_FILENAME="" +ADD ${ROOTFS_FILENAME} / # Set the profile. RUN eselect profile set "default/linux/amd64/17.1/no-multilib/systemd/merged-usr" # Replace /etc/portage/make.conf. diff --git a/build-image.sh b/build-image.sh index 7d8d636..c0a346d 100755 --- a/build-image.sh +++ b/build-image.sh @@ -91,6 +91,7 @@ podman tag rm "${REGISTRY}:latest" # Do not exit_err here. At least on first run # there is no latest tag to delete. # === Build the new image. -podman build "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." +podman build --build-arg=ROOTFS_FILENAME="${DISTFILES}/${ARCHIVE_FILE_NAME}" \ + "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." # vim:fileencoding=utf-8:ts=4:syntax=bash:expandtab From 3258794abb950bcaf10cc9f464aa917020003995 Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 22 Oct 2023 11:17:18 +0200 Subject: [PATCH 7/8] Fix image downloads --- build-image.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build-image.sh b/build-image.sh index c0a346d..d3a83a4 100755 --- a/build-image.sh +++ b/build-image.sh @@ -71,12 +71,20 @@ _mkdir "${BINPKG}" _mkdir "${LOGDIR}" # === Fetch the stage3 file (and verify it). +# Note: This uses some nasty string manipulation assuming a certain structure. +# If upstream changes the format, things will break here, again. SERVER="https://ftp-osl.osuosl.org/pub/gentoo/releases/${ARCH}/autobuilds" MY_STAGE3="latest-stage3-amd64-nomultilib-systemd-mergedusr.txt" -LATEST_ARCHIVE="$(curl -sLC- ${SERVER}/${MY_STAGE3} | tail -n1 | cut -f1 -d' ')" + +# Fetch the stage3 archive and its signature. +curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${MY_STAGE3}" \ + || exit_err "Could not download the pointer file for the stage3 archive." +gpg --verify "${DISTFILES}/${MY_STAGE3}" \ + || exit_err "Could not verify the download pointer file." + +LATEST_ARCHIVE="$(grep $(echo ${MY_STAGE3} | sed 's/latest-//;s/.txt//') ${DISTFILES}/${MY_STAGE3} | cut -f1 -d' ')" ARCHIVE_FILE_NAME="$(echo ${LATEST_ARCHIVE} | cut -f2 -d'/')" -### Fetch the stage3 archive and its signature. curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}" \ || exit_err "Could not download the stage3 archive." curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}.asc" \ From 1632623c7a55ac9eed6f5e211ab6a3ffe2b2e81b Mon Sep 17 00:00:00 2001 From: Nils Freydank Date: Sun, 22 Oct 2023 11:17:18 +0200 Subject: [PATCH 8/8] Fix stage3 import path As files can only be added from the current dir and below I decided to introduce a new DISTFILES_STAGE3 and put the stage3 tarballs fetched from upstream gentoo there. --- build-image.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/build-image.sh b/build-image.sh index d3a83a4..5524e4c 100755 --- a/build-image.sh +++ b/build-image.sh @@ -12,6 +12,7 @@ IMAGE_TAG="${REGISTRY}:${VERSION}" REPOS="${REPOS:-/var/db/repos}" DISTFILES="${DISTFILES:-/var/cache/distfiles-podman-1}" +DISTFILES_STAGE3="${DISTFILES_STAGE3:-distfiles}" BINPKG="${BINPKG:-/var/cache/packages-podman-1}" LOGDIR="${LOGDIR:-$(pwd)/log}" DOCKER_FILE="${DOCKER_FILE:-$(pwd)/Dockerfile}" @@ -67,6 +68,7 @@ _mkdir() # === Prepare all directories. _mkdir "${REPOS}" _mkdir "${DISTFILES}" +_mkdir "${DISTFILES_STAGE3}" _mkdir "${BINPKG}" _mkdir "${LOGDIR}" @@ -77,21 +79,21 @@ SERVER="https://ftp-osl.osuosl.org/pub/gentoo/releases/${ARCH}/autobuilds" MY_STAGE3="latest-stage3-amd64-nomultilib-systemd-mergedusr.txt" # Fetch the stage3 archive and its signature. -curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${MY_STAGE3}" \ +curl -sLC- -O --output-dir "${DISTFILES_STAGE3}" "${SERVER}/${MY_STAGE3}" \ || exit_err "Could not download the pointer file for the stage3 archive." -gpg --verify "${DISTFILES}/${MY_STAGE3}" \ +gpg --verify "${DISTFILES_STAGE3}/${MY_STAGE3}" \ || exit_err "Could not verify the download pointer file." -LATEST_ARCHIVE="$(grep $(echo ${MY_STAGE3} | sed 's/latest-//;s/.txt//') ${DISTFILES}/${MY_STAGE3} | cut -f1 -d' ')" +LATEST_ARCHIVE="$(grep $(echo ${MY_STAGE3} | sed 's/latest-//;s/.txt//') ${DISTFILES_STAGE3}/${MY_STAGE3} | cut -f1 -d' ')" ARCHIVE_FILE_NAME="$(echo ${LATEST_ARCHIVE} | cut -f2 -d'/')" -curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}" \ +curl -sLC- -O --output-dir "${DISTFILES_STAGE3}" "${SERVER}/${LATEST_ARCHIVE}" \ || exit_err "Could not download the stage3 archive." -curl -sLC- -O --output-dir "${DISTFILES}" "${SERVER}/${LATEST_ARCHIVE}.asc" \ +curl -sLC- -O --output-dir "${DISTFILES_STAGE3}" "${SERVER}/${LATEST_ARCHIVE}.asc" \ || exit_err "Could not download the stage3 archive signature." # Verify the signature. -gpg --verify "${DISTFILES}/${ARCHIVE_FILE_NAME}"{.asc,} \ +gpg --verify "${DISTFILES_STAGE3}/${ARCHIVE_FILE_NAME}"{.asc,} \ || exit_err "Could not verify the stage3 archive." # === Remove the old tag 'latest'. @@ -99,7 +101,7 @@ podman tag rm "${REGISTRY}:latest" # Do not exit_err here. At least on first run # there is no latest tag to delete. # === Build the new image. -podman build --build-arg=ROOTFS_FILENAME="${DISTFILES}/${ARCHIVE_FILE_NAME}" \ +podman build --build-arg=ROOTFS_FILENAME="${DISTFILES_STAGE3}/${ARCHIVE_FILE_NAME}" \ "${PODMAN_BUILD_ARGS[@]}" || exit_err "Build failed." # vim:fileencoding=utf-8:ts=4:syntax=bash:expandtab