From 6bedfcfdc3a38527d1eb01612ddf51f4b1753775 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Feb 2020 14:48:00 +0000 Subject: [PATCH 1/6] Post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 36995403..f3bdcdc8 100644 --- a/configure.ac +++ b/configure.ac @@ -7,10 +7,10 @@ dnl Seed the release notes with `git-shortlog-with-prs ..`. Th dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2020]) -m4_define([release_version], [2]) +m4_define([release_version], [3]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 24691c509c590d4e83b46bcafffd59d08b2666f1 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 2 Mar 2020 14:22:14 -0500 Subject: [PATCH 2/6] ci: migrate to new coreos-ci project Use the new custom steps. I think we could simplify things further by using `fcosBuild` but let's start with this for now. --- .cci.jenkinsfile | 75 ++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 96e52441..83f81411 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -1,47 +1,40 @@ -// See https://github.com/jlebon/coreos-ci -// This was originally copied from the rpm-ostree one -@Library('github.com/coreos/coreos-ci-lib@master') _ - -COSA_IMAGE = 'quay.io/coreos-assembler/coreos-assembler:latest' -COSA_BUILDROOT_IMAGE = 'registry.svc.ci.openshift.org/coreos/cosa-buildroot:latest' +// Documentation: https://github.com/coreos/coreos-ci/blob/master/README-upstream-ci.md stage("Build") { parallel normal: { - coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + cosaPod(buildroot: true, runAsUser: 0) { checkout scm stage("Core build") { - sh """ - set -xeuo pipefail - # fetch tags so `git describe` gives a nice NEVRA when building the RPM - git fetch origin --tags - git submodule update --init + shwrap(""" + # fetch tags so `git describe` gives a nice NEVRA when building the RPM + git fetch origin --tags + git submodule update --init - env SKIP_INSTALLDEPS=1 ./ci/build.sh - """ + env SKIP_INSTALLDEPS=1 ./ci/build.sh + """) } stage("Unit tests") { - try { - sh """ - make check - make syntax-check - """ - } finally { - archiveArtifacts allowEmptyArchive: true, artifacts: 'test-suite.log' + try { + shwrap(""" + make check + make syntax-check + """) + } finally { + archiveArtifacts allowEmptyArchive: true, artifacts: 'test-suite.log' + } } - } - sh """ + shwrap(""" make install DESTDIR=\$(pwd)/insttree/ tar -c -C insttree/ -zvf insttree.tar.gz . - """ + """) stash includes: 'insttree.tar.gz', name: 'build' } }, // A minimal build, helps test our build options minimal: { - coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + cosaPod(buildroot: true, runAsUser: 0) { checkout scm - sh """ - set -xeuo pipefail + shwrap(""" git submodule update --init env NOCONFIGURE=1 ./autogen.sh @@ -50,19 +43,18 @@ minimal: { --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse \ --disable-experimental-api make - """ + """) } }, codestyle: { - coreos.pod(image: COSA_IMAGE) { + cosaPod { checkout scm - sh """ - set -xeuo pipefail + shwrap(""" # Jenkins by default only fetches the branch it's testing. Explicitly fetch master # for ci-commitmessage-submodules.sh git fetch origin +refs/heads/master:refs/remotes/origin/master ci/ci-commitmessage-submodules.sh - """ + """) } } } @@ -70,12 +62,11 @@ codestyle: { // Build FCOS and do a kola basic run stage("More builds and test") { parallel fcos: { - coreos.pod(image: COSA_IMAGE, runAsUser: 0, kvm: true, memory: "2048Mi", cpu: "2") { + cosaPod(runAsUser: 0, memory: "2048Mi", cpu: "2") { stage("Build FCOS") { checkout scm unstash 'build' - sh """ - set -xeuo pipefail + shwrap(""" mkdir insttree tar -C insttree -xzvf insttree.tar.gz rsync -rlv insttree/ / @@ -84,23 +75,19 @@ parallel fcos: { mv insttree/* overrides/rootfs/ rmdir insttree coreos-assembler build - """ + """) } stage("FCOS basic") { timeout(time: 30, unit: 'MINUTES') { - sh """ - set -xeuo pipefail - cosa kola run --basic-qemu-scenarios - """ + shwrap("cosa kola run --basic-qemu-scenarios") } } } }, buildopts: { - coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + cosaPod(buildroot: true, runAsUser: 0) { checkout scm - sh """ - set -xeuo pipefail + shwrap(""" git submodule update --init git worktree add build-rust && cd build-rust @@ -112,7 +99,7 @@ buildopts: { env CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh make check cd .. && rm -rf build-libsoup - """ + """) } } } From 0302262e2faac893bb4e4b5497e11ca28e932398 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 13 Mar 2020 17:29:58 +0000 Subject: [PATCH 3/6] ci: Test kola --upgrades This one is very relevant for ostree. --- .cci.jenkinsfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 83f81411..a83adeea 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -82,6 +82,15 @@ parallel fcos: { shwrap("cosa kola run --basic-qemu-scenarios") } } + stage("FCOS upgrades") { + timeout(time: 30, unit: 'MINUTES') { + shwrap(""" + cosa kola --upgrades --no-test-exit-error + tar -cf - tmp/kola-upgrade | xz -c9 > kola-run-upgrade.tar.xz + """) + archiveArtifacts "kola-run-upgrade.tar.xz" + } + } } }, buildopts: { From cbb3f0a8c2265a0124f94296f4780a51ffce3a22 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 13 Mar 2020 14:35:53 -0400 Subject: [PATCH 4/6] ci: use `fcosKola` for running kola tests We were using `--no-test-exit-error` for upgrade tests but weren't actually checking for test failures after. Instead of running kola directly, just use the `fcosKola` custom step which automatically takes care of e.g. running tests in parallel and archiving results. --- .cci.jenkinsfile | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index a83adeea..eaa976f5 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -77,20 +77,7 @@ parallel fcos: { coreos-assembler build """) } - stage("FCOS basic") { - timeout(time: 30, unit: 'MINUTES') { - shwrap("cosa kola run --basic-qemu-scenarios") - } - } - stage("FCOS upgrades") { - timeout(time: 30, unit: 'MINUTES') { - shwrap(""" - cosa kola --upgrades --no-test-exit-error - tar -cf - tmp/kola-upgrade | xz -c9 > kola-run-upgrade.tar.xz - """) - archiveArtifacts "kola-run-upgrade.tar.xz" - } - } + fcosKola("${env.WORKSPACE}") } }, buildopts: { From 6a27dff94100c31edf392b26ace05009f4dedf9c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 Mar 2020 21:10:29 +0000 Subject: [PATCH 5/6] main: Also automatically remount rw /sysroot for `ostree pull` etc. See https://github.com/coreos/fedora-coreos-tracker/issues/343 When we added the read-only sysroot support it broke using "raw" `ostree pull` and `ostree refs --create` and all of the core repo CLIs that just operate on a repo and not a sysroot. Fixing this is a bit ugly as it "layer crosses" things even more. Extract a helper function that works in both cases. --- src/ostree/ot-main.c | 84 +++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 21 deletions(-) diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index a044cef2..bffa40c4 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include "ot-main.h" #include "ostree.h" @@ -100,6 +102,49 @@ ostree_usage (OstreeCommand *commands, return (is_error ? 1 : 0); } +/* If we're running as root, booted into an OSTree system and have a read-only + * /sysroot, then assume we may need write access. Create a new mount namespace + * if so, and return *out_ns = TRUE. Otherwise, *out_ns = FALSE. + */ +static gboolean +maybe_setup_mount_namespace (gboolean *out_ns, + GError **error) +{ + *out_ns = FALSE; + + /* If we're not root, then we almost certainly can't be remounting anything */ + if (getuid () != 0) + return TRUE; + + /* If the system isn't booted via libostree, also nothing to do */ + if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) + return FALSE; + if (errno == ENOENT) + return TRUE; + + glnx_autofd int sysroot_subdir_fd = glnx_opendirat_with_errno (AT_FDCWD, "/sysroot", TRUE); + if (sysroot_subdir_fd < 0) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "opendirat"); + /* No /sysroot - nothing to do */ + return TRUE; + } + + struct statvfs stvfs; + if (fstatvfs (sysroot_subdir_fd, &stvfs) < 0) + return glnx_throw_errno_prefix (error, "fstatvfs"); + if (stvfs.f_flag & ST_RDONLY) + { + if (unshare (CLONE_NEWNS) < 0) + return glnx_throw_errno_prefix (error, "preparing writable sysroot: unshare (CLONE_NEWNS)"); + + *out_ns = TRUE; + } + + return TRUE; +} + static void message_handler (const gchar *log_domain, GLogLevelFlags log_level, @@ -220,6 +265,19 @@ parse_repo_option (GOptionContext *context, { g_autoptr(OstreeRepo) repo = NULL; + /* This is a bit of a brutal hack; we set up a mount + * namespace if it appears that we may need it. It'd + * be better to do this more precisely in the future. + */ + gboolean setup_ns = FALSE; + if (!maybe_setup_mount_namespace (&setup_ns, error)) + return FALSE; + if (setup_ns) + { + if (mount ("/sysroot", "/sysroot", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) + return glnx_null_throw_errno_prefix (error, "Remounting /sysroot read-write"); + } + if (repo_path == NULL) { g_autoptr(GError) local_error = NULL; @@ -452,27 +510,11 @@ ostree_admin_option_context_parse (GOptionContext *context, */ if (ostree_sysroot_is_booted (sysroot)) { - int sysroot_fd = ostree_sysroot_get_fd (sysroot); - g_assert_cmpint (sysroot_fd, !=, -1); - - glnx_autofd int sysroot_subdir_fd = glnx_opendirat_with_errno (sysroot_fd, "sysroot", TRUE); - if (sysroot_subdir_fd < 0) - { - if (errno != ENOENT) - return glnx_throw_errno_prefix (error, "opendirat"); - } - else if (getuid () == 0) - { - struct statvfs stvfs; - if (fstatvfs (sysroot_subdir_fd, &stvfs) < 0) - return glnx_throw_errno_prefix (error, "fstatvfs"); - if (stvfs.f_flag & ST_RDONLY) - { - if (unshare (CLONE_NEWNS) < 0) - return glnx_throw_errno_prefix (error, "preparing writable sysroot: unshare (CLONE_NEWNS)"); - ostree_sysroot_set_mount_namespace_in_use (sysroot); - } - } + gboolean setup_ns = FALSE; + if (!maybe_setup_mount_namespace (&setup_ns, error)) + return FALSE; + if (setup_ns) + ostree_sysroot_set_mount_namespace_in_use (sysroot); } /* Released when sysroot is finalized, or on process exit */ From 6ed48234ba579ff73eb128af237212b0a00f2057 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 13 Mar 2020 21:54:21 -0400 Subject: [PATCH 6/6] Release 2020.3 Let's do another release to get the `sysroot.readonly` fixes into FCOS and unpin ostree and rpm-ostree there. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f3bdcdc8..f8d7b2be 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ m4_define([year_version], [2020]) m4_define([release_version], [3]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux])