Import from https://gitlab.com/CentOS/automotive/sample-images

Commit: a95b833c

tinyrepo/README.rst

@@ -0,0 +1,31 @@
+Tinyrepo
+========
+
+This folder contains some utility scripts that are used to sample images that
+can be easily downloaded and used.
+
+## Creating sample images
+
+The sample images are a sub-set of the images one can build using the osbuild
+manifest present in this repository.
+
+The creating of the images is handled by the `build_images.sh` shell script
+which simply calls sequentially different `make` commands, then compress the
+resulting images and places them in a timestamped directory available to apache.
+Finally, it creates a symlink between the newest folder and a `latest` folder,
+thus giving a stable location where one can find the most recently built images.
+
+
+## Cron job
+
+The `build_images.sh` shell script present in this folder are ran via the
+following cron job configuration:
+
+::
+
+  0 2 * * 1 sh /path/to/automotive-sig/tinyrepo/build_images.sh /path/to/clone/of/automotive-sig/osbuild-manifests/ /path/to/root/folder/in/apache
+
+
+This allows to build new images weekly.
+
+

tinyrepo/build_images.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+d=`date "+%Y%m%d"`
+AUTOSIGREPO=$1
+HTMLROOT=$2
+
+pushd $AUTOSIGREPO/osbuild-manifests
+
+# Let's update the git repo
+git pull --rebase
+
+# Clean all all cached data
+make clean
+rm *.log || true
+rm *.img || true
+rm *.qcow2 || true
+rm *.img.xz || true
+rm *.qcow2.xz || true
+dnf clean all
+sudo dnf clean all
+
+# Build all images
+sudo make cs9-rpi4-neptune-ostree.aarch64.img > cs9-rpi4-neptune-ostree.aarch64.log
+sudo make cs9-rpi4-neptune-regular.aarch64.img > cs9-rpi4-neptune-regular.aarch64.log
+sudo make cs9-qemu-developer-regular.aarch64.qcow2 > cs9-qemu-developer-regular.aarch64.log
+sudo make cs9-qemu-developer-regular.aarch64.oci.tar > cs9-qemu-developer-regular.aarch64.oci.tar.log
+sudo make osbuildvm-images > osbuildvm-images.log
+
+# Compact them
+xz cs9-rpi4-neptune-ostree.aarch64.img &
+xz cs9-rpi4-neptune-regular.aarch64.img &
+xz cs9-qemu-developer-regular.aarch64.qcow2 &
+tar cfz osbuildvm-images.tar.gz _build/osbuildvm-aarch64.* &
+
+wait
+
+popd
+
+# Move the images to apache
+mkdir -p $HTMLROOT/images/$d
+cp $AUTOSIGREPO/osbuild-manifests/*.xz  $HTMLROOT/images/$d/
+cp $AUTOSIGREPO/osbuild-manifests/*.tar  $HTMLROOT/images/$d/
+cp $AUTOSIGREPO/osbuild-manifests/*.tar.gz  $HTMLROOT/images/$d/
+
+# Update the `latest` symlink
+ln -nsf $d $HTMLROOT/images/latest

tinyrepo/build_repos.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+d=`date "+%Y%m%d"`
+AUTOSIGREPO=$1
+HTMLROOT=$2
+
+python $AUTOSIGREPO/tinyrepo/initiate_repo.py \
+    $HTMLROOT/cs9-stream/ \
+    --pkgslist=$HTMLROOT/cs9-stream/pkgslist_$d
+
+for arch in aarch64 x86_64
+do
+  mkdir -p $HTMLROOT/cs9-stream/$arch/$d
+  pushd $HTMLROOT/cs9-stream/$arch/$d
+  createrepo_c ../packages/ -o . -i ../../pkgslist_$d
+  popd
+
+  # Update the `latest` symlink
+  ln -nsf $d $HTMLROOT/cs9-stream/$arch/latest
+done

tinyrepo/initiate_repo.py

@@ -0,0 +1,145 @@
+#!/usr/bin/python3
+
+"""
+This script queries content resolver at: https://tiny.distro.builders to retrieve
+a list of all the packages of interest for the automotive SIG.
+
+Requirements:
+  - python-requests
+
+"""
+
+import argparse
+import os
+import re
+import sys
+
+import requests
+
+
+environments = [
+    "https://tiny.distro.builders/workload--automotive-c9s-workload-development--automotive-c9s-env-minimum--automotive-c9s-repositories--aarch64.json",
+    "https://tiny.distro.builders/workload--automotive-c9s-workload-minimum--automotive-c9s-env-minimum--automotive-c9s-repositories--aarch64.json",
+    "https://tiny.distro.builders/workload--automotive-c9s-workload-neptune--automotive-c9s-env-minimum--automotive-c9s-repositories--aarch64.json",
+    "https://tiny.distro.builders/workload--automotive-c9s-workload-ostree--automotive-c9s-env-minimum--automotive-c9s-repositories--aarch64.json",
+    "https://tiny.distro.builders/workload--automotive-c9s-workload-podman--automotive-c9s-env-minimum--automotive-c9s-repositories--aarch64.json",
+]
+
+cs_urls = [
+    "http://mirror.stream.centos.org/9-stream/AppStream/{arch}/os/Packages/{nvr}",
+    "http://mirror.stream.centos.org/9-stream/BaseOS/{arch}/os/Packages/{nvr}",
+    "http://mirror.stream.centos.org/9-stream/CRB/{arch}/os/Packages/{nvr}",
+    "https://composes.stream.centos.org/development/latest-CentOS-Stream/compose/BaseOS/{arch}/os/Packages/{nvr}",
+    "https://composes.stream.centos.org/development/latest-CentOS-Stream/compose/AppStream/{arch}/os/Packages/{nvr}",
+    "https://composes.stream.centos.org/development/latest-CentOS-Stream/compose/CRB/{arch}/os/Packages/{nvr}",
+]
+
+epoch_re = re.compile("-\d+:")
+
+
+def get_packages():
+    """ Queries content resolver and returns a tuple of two set, the first
+    set contains the name of every package, the second set contains the NVR
+    of every package.
+    """
+    packages_set = set()
+    nvr_set = set()
+    for env in environments:
+        req = requests.get(env)
+        if not req.ok:
+            print(f"Failed to retrieve info from {env}")
+            continue
+        data = req.json()
+        data = data["data"]
+        for key in ("pkg_env_ids", "pkg_added_ids"):
+            for pkg in data[key]:
+                name = pkg.rsplit("-", 2)[0]
+                # Drop the architecture
+                nvr = pkg.rsplit(".", 1)[0]
+                # Drop epoch if there is one
+                if len(epoch_re.findall(nvr)) == 1:
+                    for mat in epoch_re.findall(nvr):
+                        nvr = nvr.replace(mat, "-")
+                elif len(epoch_re.findall(nvr)) > 1:
+                    print("More than one -\d: matched in {nvr}")
+                packages_set.add(name)
+                nvr_set.add(nvr)
+
+    return (packages_set, nvr_set)
+
+
+def download_nvrs(folder, nvrs):
+    """ Queries the CentOS-Stream mirror and download the package specified by
+    the given NVR.
+    """
+    notfound = 0
+    out_files = []
+    for nvr in sorted(nvrs):
+        for arch in ("x86_64", "aarch64"):
+            out_filename = f"{nvr}.{arch}.rpm"
+            output_folder = os.path.join(folder, arch, "packages")
+            out_path = os.path.join(output_folder, out_filename)
+
+            if not os.path.exists(output_folder):
+                os.makedirs(output_folder)
+
+            if os.path.exists(out_path):
+                out_files.append(out_filename)
+                continue
+
+            found = False
+            for url in cs_urls:
+                nurl = f"{url}.{arch}.rpm".format(arch=arch, nvr=nvr)
+                req = requests.head(nurl)
+                # print(req.status_code, url)
+                if req.ok:
+                    found = True
+                    out_files.append(out_filename)
+                    req = requests.get(nurl)
+                    with open(out_path, "wb") as stream:
+                        stream.write(req.content)
+                    break
+                else:
+                    # print(req.status_code, nurl)
+                    # Check if it's a noarch RPM
+                    nurl = f"{url}.noarch.rpm".format(arch=arch, nvr=nvr)
+                    req = requests.head(nurl)
+                    if req.ok:
+                        found = True
+                        out_files.append(f"{nvr}.noarch.rpm")
+                        req = requests.get(nurl)
+                        with open(out_path, "wb") as stream:
+                            stream.write(req.content)
+                        break
+                    # else:
+                        # print(req.status_code, nurl)
+            if not found:
+                print(f"Package {nvr}.{arch} not found on the mirrors")
+                notfound += 1
+
+    print(f"{notfound} packages not found on the mirrors")
+    return out_files
+
+
+def parse_arguments(args):
+    """Parse the CLI arguments and return the corresponding argparse object.
+    """
+    parser = argparse.ArgumentParser(description="Instantiate george")
+    parser.add_argument("folder", nargs="?", default=".",
+                        help="Folder in which to put the downloaded RPMs")
+    parser.add_argument("--pkgslist",
+                        help="Name of the file in which to output the list of all the RPMs found")
+
+    return parser.parse_args(args)
+
+
+if __name__ == "__main__":
+    args = parse_arguments(sys.argv[1:])
+    pkgs, nvrs = get_packages()
+    # print("-", "\n- ".join(sorted(pkgs)))
+    print(len(pkgs), "packages found")
+    print("Corresponding to", len(nvrs), "nvr")
+    out_files = download_nvrs(args.folder, nvrs)
+    if args.pkgslist:
+        with open(args.pkgslist, "w") as stream:
+            stream.write("\n".join(out_files))