Merge branch 'custom-base-target' into 'main'

Draft: Add bootc-base-imagectl onto the main branch

See merge request fedora/bootc/base-images!98

.gitlab-ci.yml

@@ -1,14 +1,32 @@
 ---
-include:
-  - remote: https://gitlab.com/platform-engineering-org/gitlab-ci/-/raw/main/templates/build-image.gitlab-ci.yml
+stages:
+  - build
+
+variables:
+  IMAGE: ${CI_REGISTRY}/${CI_PROJECT_PATH}:${CI_COMMIT_SHA}
+  CONTAINERFILE: Containerfile
+  CONTEXT: .
+  EXTRA_ARGS: ""
+
+.build-image:
+  stage: build
+  image: quay.io/buildah/stable:v1.38.0
+  needs: []
+  script: buildah bud -f ${CONTAINERFILE} --no-cache -t ${IMAGE} ${EXTRA_ARGS} ${CONTEXT}
+  rules:
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
 
 build-image:
   extends: .build-image
   parallel:
     matrix:
-      - TIER: [tier-0, tier-1, tier-x]
+      - TIER: [minimal, standard, tier-x]
   variables:
-    EXTRA_ARGS: "--security-opt=label=disable --cap-add=all --build-arg MANIFEST=fedora-$TIER.yaml"
+    EXTRA_ARGS: "--security-opt=label=disable --cap-add=all --build-arg MANIFEST=fedora-$TIER"
   rules:
     - if: $CI_PROJECT_NAMESPACE != "fedora/bootc"
       when: never

Containerfile

@@ -1,36 +1,17 @@
-# This container build uses some special features of podman that allow
-# a process executing as part of a container build to generate a new container
-# image "from scratch".
-#
-# This container build uses nested containerization, so you must build with e.g.
+# In order to make a base image as part of a Dockerfile, this container build uses
+# nested containerization, so you must build with e.g.
 # podman build --security-opt=label=disable --cap-add=all --device /dev/fuse <...>
-#
-# # Why are we doing this?
-#
-# Today this base image build process uses rpm-ostree.  There is a lot of things that
-# rpm-ostree does when generating a container image...but important parts include:
-#
-# - auto-updating labels in the container metadata
-# - Generating "chunked" content-addressed reproducible image layers (notice
-#   how there are ~60 layers in the generated image)
-#
-# The latter bit in particular is currently impossible to do from Containerfile.
-# A future goal is adding some support for this in a way that can be honored by
-# buildah (xref https://github.com/containers/podman/discussions/12605)
-#
-# # Why does this build process require additional privileges?
-#
-# Because it's generating a base image and uses containerization features itself.
-# In the future some of this can be lifted.
+
+# NOTE: This container build will output a single giant layer. It is strongly recommended
+# to run the "rechunker" on the output of this build, see 
+# https://coreos.github.io/rpm-ostree/experimental-build-chunked-oci/
 
 FROM quay.io/fedora/fedora:rawhide as repos
 
 # BOOTSTRAPPING: This can be any image that has rpm-ostree and selinux-policy-targeted.
 FROM quay.io/fedora/fedora:rawhide as builder
 RUN dnf -y install rpm-ostree selinux-policy-targeted
-ARG MANIFEST=fedora-bootc.yaml
-COPY --from=repos /etc/dnf/vars /etc/dnf/vars
-COPY --from=repos /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-* /etc/pki/rpm-gpg
+ARG MANIFEST=fedora-standard
 # The input git repository has .repo files committed to git rpm-ostree has historically
 # emphasized that.  But here, we are fetching the repos from the container base image.
 # So copy the source, and delete the hardcoded ones in git, and use the container base
@@ -38,17 +19,29 @@ COPY --from=repos /etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-* /etc/pki/rpm-gpg
 COPY . /src
 WORKDIR /src
 RUN rm -vf /src/*.repo
-COPY --from=repos /etc/yum.repos.d/*.repo /src
 RUN --mount=type=cache,target=/workdir \
-    --mount=type=bind,rw=true,src=.,dst=/buildcontext,bind-propagation=shared \
-    --mount=type=bind,from=repos,src=/,dst=/repos \
-      rpm-ostree compose image --image-config fedora-bootc-config.json \
-        --cachedir=/workdir --format=ociarchive --initialize ${MANIFEST} \
-       --source-root=/repos /buildcontext/out.ociarchive
+    --mount=type=bind,rw,from=repos,src=/,dst=/repos <<EORUN
+set -xeuo pipefail
+# Put our manifests into the builder image in the same location they'll be in the
+# final image.
+./install-manifests
+# Verify that listing works
+/usr/libexec/bootc-base-imagectl list >/dev/null
+# Run the build script in the same way we expect custom images to do, and also
+# "re-inject" the manifests into the target, so secondary container builds can use it.
+/usr/libexec/bootc-base-imagectl build-rootfs --reinject --manifest=${MANIFEST} /repos /target-rootfs
+EORUN
 
-FROM oci-archive:./out.ociarchive
-# Need to reference builder here to force ordering. But since we have to run
-# something anyway, we might as well cleanup after ourselves.
-RUN --mount=type=bind,from=builder,src=.,target=/var/tmp \
-    --mount=type=bind,rw=true,src=.,dst=/buildcontext,bind-propagation=shared \
-      rm /buildcontext/out.ociarchive
+# This pulls in the rootfs generated in the previous step
+FROM scratch
+COPY --from=builder /target-rootfs/ /
+LABEL containers.bootc 1
+# This is an ad-hoc way for us to reference bootc-image-builder in
+# a way that in theory client tooling can inspect and find. Today
+# it isn't widely used.
+LABEL bootc.diskimage-builder quay.io/centos-bootc/bootc-image-builder
+# https://pagure.io/fedora-kiwi-descriptions/pull-request/52
+ENV container=oci
+# Make systemd the default
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

README.md

@@ -30,7 +30,7 @@ podman build --security-opt=label=disable --cap-add=all \
   --device /dev/fuse -t localhost/fedora-bootc .
 ```
 
-See the `Containerfile` for more details. This builds the default `tier-1` image.
+See the `Containerfile` for more details. This builds the default `standard` image.
 
 ## Fedora versions
 
@@ -58,21 +58,20 @@ It is planned to rework and improve this in the future, especially
 to support smaller custom images. For more on this, see
 [this tracker issue](https://gitlab.com/fedora/bootc/tracker/-/issues/32).
 
-- **tier-1**: This image is the default, what is published as
+- **standard**: This image is the default, what is published as
   https://quay.io/repository/fedora/fedora-bootc
-- **tier-0**: This content set is more of a convenient centralization point for CI
-  and curation around a package set that we can all agree is the rough minimum
-  necessary for a usable system. It's not meant to be used as is, but layered
-  upon.
+- **minimal**: This content set is more of a convenient centralization point for CI
+  and curation around a package set that is intended as a starting point fror
+  a container base image.
 - **tier-x**: This content set is the shared base used by all image-based
   Fedora variants (IoT, Atomic Desktops, and CoreOS).
   Changes to this tier may be done without accounting for external users.
   To build this, pass `--build-arg=MANIFEST=fedora-tier-x.yaml` to the build
   command above.
 
-**tier-1** inherits from **tier-x** and **tier-x** in turn inherit from **tier-0**.
+**standard** inherits from **tier-x** and **tier-x** in turn inherit from **minimal**.
 
-All non-trivial changes to **tier-0** and **tier-x** should be ACKed by at least
+All non-trivial changes to **minimal** and **tier-x** should be ACKed by at least
 one stakeholder of each Fedora variant WGs.
 
 ## More information

bootc-base-imagectl

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+import os
+import os.path as path
+import subprocess
+import shutil
+import json
+import argparse
+import sys
+
+MANIFESTDIR = 'usr/share/doc/bootc-base-imagectl/manifests'
+
+def run_build_rootfs(args):
+    """
+    Regenerates a base image using a build configuration.
+    """
+    target = args.target
+    if os.path.isdir(args.manifest):
+        manifest_path = os.path.join(args.manifest, 'manifest.yaml')
+    else:
+        manifest_path = args.manifest + '.yaml'
+    try:
+        # Perform the build
+        subprocess.run([
+            'rpm-ostree',
+            'experimental',
+            'compose',
+            'rootfs',
+            f'--source-root-rw={args.source_root}',
+            f'/{MANIFESTDIR}/{manifest_path}',
+            target,
+        ], check=True)
+        # And run the bootc linter for good measure
+        subprocess.run([
+            'bootc',
+            'container',
+            'lint',
+            f'--rootfs={target}',
+        ], check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error executing command: {e}")
+        sys.exit(1)
+
+    # Copy our own build configuration into the target if configured;
+    # this is used for the first stage build. But by default *secondary*
+    # builds don't get this.
+    if args.reinject:
+        for d in [MANIFESTDIR]:
+            dst = path.join(target, d)
+            print(f"Copying /{d} to {dst}")
+            shutil.copytree('/' + d, dst)
+        for f in ['usr/libexec/bootc-base-imagectl']:
+            dst = path.join(target, f)
+            print(f"Copying /{f} to {dst}")
+            shutil.copy('/' + f, dst)
+
+def run_list(args):
+    d = '/' + MANIFESTDIR
+    for ent in sorted(os.listdir(d)):
+        name, ext = os.path.splitext(ent)
+        if ext != '.yaml':
+            continue
+        fullpath = os.path.join(d, ent)
+        if os.path.islink(fullpath):
+            continue
+        o = subprocess.check_output(['rpm-ostree', 'compose', 'tree', '--print-only', fullpath])
+        manifest = json.loads(o)
+        description = manifest['metadata']['summary']
+        print(f"{name}: {description}")
+        print("---")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Operate on the build configuration for this container")
+    subparsers = parser.add_subparsers(help='Subcommands', required=True)
+
+    build_rootfs = subparsers.add_parser('build-rootfs', help='Generate a container root filesystem')
+    build_rootfs.add_argument("--reinject", help="Also reinject the build configurations into the target", action='store_true')
+    build_rootfs.add_argument("--manifest", help="Use the specified manifest", action='store', default='default')
+    build_rootfs.add_argument("source_root", help="Path to the source root directory used for dnf configuration.")
+    build_rootfs.add_argument("target", help="Path to the target root directory that will be generated.")
+    build_rootfs.set_defaults(func=run_build_rootfs)
+
+    cmd_list = subparsers.add_parser('list', help='List available manifests')
+    cmd_list.set_defaults(func=run_list)
+
+    args = parser.parse_args()
+    args.func(args)

bootc-base-imagectl.md

@@ -0,0 +1,99 @@
+# bootc-base-imagectl
+
+A core premise of the bootc model is that rich
+control over Linux system customization can be accomplished
+with a "default" container build:
+
+```
+FROM <base image>
+RUN ...
+```
+
+As of recently, it is possible to e.g. swap the kernel
+and other fundamental components as part of default derivation.
+
+However, some use cases want even more control - for example,
+as an organization deploying a bootc system, I may want to ensure
+the base image version carries a set of packages at
+exactly specific versions (perhaps defined by a lockfile,
+or an rpm-md repository). There are many tools which
+manage snapshots of yum (rpm-md) repositories.
+
+There are currently issues where it won't quite work to e.g.
+`dnf -y upgrade selinux-policy-targeted`.
+
+The `/usr/libexec/bootc-base-imagectl` tool which is
+included in the base image is designed to enable building
+a root filesystem in ostree-container format from a set
+of RPMs controlled by the user.
+
+## Understanding the base image content
+
+Most, but not all content from the base image comes from RPMs.
+There is some additional non-RPM content, as well as postprocessing
+that operates on the filesystem root. At the current time the
+implementation of the base image build uses `rpm-ostree`,
+but this is considered an implementation detail subject to change.
+
+## Using bootc-base-imagectl build-rootfs
+
+The core operation is `bootc-base-imagectl build-rootfs`.
+
+This command takes just two arguments:
+
+- A "source root" which should have an `/etc/yum.repos.d`
+  that defines the input RPM content. This source root is also used
+  to control things like the `$releasever`.
+- A path to the target root filesystem which will be generated as
+  a directory. The target should not already exist (but its parent must exist).
+
+### Other options
+
+`bootc-base-imagectl list` will enumerate available configurations that
+can be selected by passing `--manifest` to `build-rootfs`.
+
+### Implementation
+
+The current implementation uses `rpm-ostree` on a manifest (treefile)
+embedded in the container image itself. These manifests are not intended
+to be editable directly. 
+
+To emphasize: the implementation of this command (especially the configuration
+files that it reads) are subject to change.
+
+### Cross builds and the builder image
+
+The build tooling is designed to support "cross builds"; the
+repository root could e.g. be CentOS Stream 10, while the
+builder root is Fedora or RHEL, etc. 
+
+In other words, one given base image can be used as a "builder" to produce another
+using different RPMs.
+
+### Example: Generate a new image using CentOS Stream 10 content from RHEL
+
+FROM quay.io/centos/centos:stream10 as repos
+
+FROM registry.redhat.io/rhel10/rhel-bootc:10 as builder
+RUN --mount=type=bind,from=repos,src=/,dst=/repos,rw /usr/libexec/bootc-base-imagectl build-rootfs --manifest=minimal /repos /target-rootfs
+
+# This container image uses the "artifact pattern"; it has some
+# basic configuration we expect to apply to multiple container images.
+FROM quay.io/exampleos/baseconfig@sha256:.... as baseconfig
+
+FROM scratch
+COPY --from=builder /target-rootfs/ /
+# Now we make other arbitrary changes. Copy our systemd units and
+# other tweaks from the baseconfig container image.
+COPY --from=baseconfig /usr/ /usr/
+RUN <<EORUN
+set -xeuo pipefail
+# Install critical components
+dnf -y install linux-firmware NetworkManager cloud-init cowsay
+dnf clean all
+bootc container lint
+EORUN
+LABEL containers.bootc 1
+ENV container=oci
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

fedora-bootc.yaml

@@ -1,8 +0,0 @@
-metadata:
-  name: fedora-boot-tier1
-  summary: Fedora Bootable Tier 1
-
-include:
-  - fedora-generic.yaml
-  - tier-1/manifest.yaml
-  - tier-1/kernel.yaml

fedora-bootc.yaml

@@ -0,0 +1 @@
+fedora-standard.yaml

fedora-minimal.yaml

@@ -0,0 +1,3 @@
+include:
+  - fedora-includes/generic.yaml
+  - minimal/manifest.yaml

fedora-standard.yaml

@@ -0,0 +1,3 @@
+include:
+  - fedora-includes/generic.yaml
+  - standard/manifest.yaml

fedora-tier-0.yaml

@@ -1,8 +0,0 @@
-metadata:
-  name: fedora-boot-tier0
-  summary: Fedora Bootable Tier 0
-
-include:
-  - fedora-generic.yaml
-  - tier-0/manifest.yaml
-  - tier-0/kernel.yaml

fedora-tier-1.yaml

@@ -1 +0,0 @@
-fedora-bootc.yaml

fedora-tier-x.yaml

@@ -1,8 +1,3 @@
-metadata:
-  name: fedora-boot-tier-x
-  summary: Fedora Bootable Tier X
-
 include:
-  - fedora-generic.yaml
+  - fedora-includes/generic.yaml
   - tier-x/manifest.yaml
-  - tier-x/kernel.yaml

install-manifests

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -xeuo pipefail
+# This script copies the manifests from the current directory
+# into their installed location.
+manifestdir=/usr/share/doc/bootc-base-imagectl/manifests
+mkdir -p "$manifestdir/"
+for image in minimal standard tier-x; do
+    # Embed the generic defaults
+    cp -a $image $manifestdir/
+    # And the Fedora-specific tweaks
+    cp -a fedora-$image.yaml $manifestdir/
+done
+# Set the default
+ln -s fedora-standard.yaml $manifestdir/default.yaml
+# And install dependency manifests
+cp -a fedora-includes $manifestdir
+# And embed the rebuild script
+install -m 0755 -t /usr/libexec ./bootc-base-imagectl

minimal/manifest.yaml

@@ -1,3 +1,6 @@
+metadata:
+  summary: Effectively just bootc, systemd, kernel, and dnf as a starting point.
+
 edition: "2024"
 
 # Be minimal
@@ -15,6 +18,7 @@ remove-from-packages:
   - [systemd-udev, /usr/lib/systemd/system-generators/systemd-gpt-auto-generator]
 
 include:
+  - kernel.yaml
   - postprocess-conf.yaml
   - bootc.yaml
   - bootupd.yaml
@@ -30,7 +34,7 @@ packages:
   # in dnf5. In CentOS/RHEL, this pulls in dnf(4). We can simplify this back to
   # just `dnf` once the `dnf` package is retired from Fedora.
   - /usr/bin/dnf
-  # Even in tier-0, we have this.  If you don't want SELinux today, you'll need
+  # Even in minimal, we have this.  If you don't want SELinux today, you'll need
   # to build a custom image.
   - selinux-policy-targeted
   # And we want container-selinux because trying to layer it on later currently causes issues.

minimal/postprocess-conf.yaml

@@ -7,7 +7,7 @@ opt-usrlocal: "root"
 machineid-compat: true
 
 # Note that the default for c9s+ is sqlite; we can't rely on rpm being
-# in the target (it isn't in tier-0!) so turn this to host here.  This
+# in the target (it isn't in minimal!) so turn this to host here.  This
 # does break the "hermetic build" aspect a bit.  Maybe eventually
 # what we should do is special case this and actually install RPM temporarily
 # and then remove it...

standard/initramfs-full.yaml

@@ -1,8 +1,8 @@
-# Configuration for the "tier-1" initramfs
+# Configuration for the initramfs
 postprocess:
   - |
     #!/usr/bin/env bash
     mkdir -p /usr/lib/dracut/dracut.conf.d
-    cat > /usr/lib/dracut/dracut.conf.d/30-bootc-tier-1.conf << 'EOF'
+    cat > /usr/lib/dracut/dracut.conf.d/30-bootc-standard.conf << 'EOF'
     add_dracutmodules+=" lvm crypt fips "
     EOF

standard/manifest.yaml

@@ -1,3 +1,9 @@
+metadata:
+  summary: |
+    A relatively full, but still generic base image. Roughly
+    similar to a headless server installation. Automatic updates
+    are on by default.
+
 # Flip this back on, we're going to be a larger system
 recommends: true
 

tests/Containerfile.test-c10s

@@ -0,0 +1,31 @@
+# This test case exercises using the fedora-bootc image as a builder
+# to generate a minimal target image derived from CentOS Stream 10 content,
+# and then further extends it in a secondary phase.
+FROM quay.io/centos/centos:stream10 as repos
+
+# This is intentionally a locally built image
+FROM localhost/fedora-bootc as builder
+RUN --mount=type=bind,from=repos,src=/,dst=/repos,rw /usr/libexec/bootc-base-imagectl build-rootfs --manifest=standard/manifest /repos /target-rootfs
+
+# This pulls in the rootfs generated in the previous step
+FROM scratch
+COPY --from=builder /target-rootfs/ /
+RUN <<EORUN
+set -xeuo pipefail
+# Verify we have CentOS content
+. /usr/lib/os-release
+test "$ID" = centos
+
+# And install a package
+dnf -y install strace
+dnf clean all
+
+# Cleanup and lint
+rm /var/log /var/cache/* /var/lib/dnf
+bootc container lint
+EORUN
+LABEL containers.bootc 1
+ENV container=oci
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]
+

tier-1/kernel.yaml

@@ -1 +0,0 @@
-../tier-0/kernel.yaml

tier-x/kernel.yaml

@@ -1 +0,0 @@
-../tier-0/kernel.yaml

tier-x/manifest.yaml

@@ -1,5 +1,13 @@
+metadata:
+  summary: |
+    A relatively full, but still generic base image. Roughly
+    similar to a smaller Fedora CoreOS. Includes NetworkManager,
+    openssh, various CLI tools, etc.
+    
+    Automatic updates are not on by default.
+
 include:
-  - ../tier-0/manifest.yaml
+  - ../minimal/manifest.yaml
 
 packages:
   # Used by admins interactively