From c820a6133b859b66e3a8adfc15e543e503672166 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 9 Dec 2019 16:18:28 +0000 Subject: [PATCH 01/49] Post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 46a900f5..4a752e55 100644 --- a/configure.ac +++ b/configure.ac @@ -7,10 +7,10 @@ dnl Seed the release notes with `git-shortlog-with-prs ..`. Th dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2019]) -m4_define([release_version], [6]) +m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 20daf9688006f17e11af7692d2e43292b36df71f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 5 Sep 2019 00:52:15 +0000 Subject: [PATCH 02/49] finalize-staged: Use the core option parsing to load sysroot Prep for using the default mount namespace handling there that will land as part of the read-only `/sysroot` and `/boot` work. See https://github.com/ostreedev/ostree/issues/1265 --- src/ostree/ot-admin-builtin-finalize-staged.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-admin-builtin-finalize-staged.c b/src/ostree/ot-admin-builtin-finalize-staged.c index 6740f82a..3cea1bdb 100644 --- a/src/ostree/ot-admin-builtin-finalize-staged.c +++ b/src/ostree/ot-admin-builtin-finalize-staged.c @@ -34,6 +34,10 @@ #include "ostree-cmdprivate.h" #include "ostree.h" +static GOptionEntry options[] = { + { NULL } +}; + /* Called by ostree-finalize-staged.service, and in turn * invokes a cmdprivate function inside the shared library. */ @@ -46,11 +50,13 @@ ot_admin_builtin_finalize_staged (int argc, char **argv, OstreeCommandInvocation if (fstatat (AT_FDCWD, "/run/ostree-booted", &stbuf, 0) < 0) return TRUE; - g_autoptr(GFile) sysroot_file = g_file_new_for_path ("/"); - g_autoptr(OstreeSysroot) sysroot = ostree_sysroot_new (sysroot_file); - - if (!ostree_sysroot_load (sysroot, cancellable, error)) + g_autoptr(GOptionContext) context = g_option_context_new (""); + g_autoptr(OstreeSysroot) sysroot = NULL; + if (!ostree_admin_option_context_parse (context, options, &argc, &argv, + OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, + invocation, &sysroot, cancellable, error)) return FALSE; + if (!ostree_cmd__private__()->ostree_finalize_staged (sysroot, cancellable, error)) return FALSE; From 5af403be0cc64df50ad21cef05f3268ead256d6d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 3 Oct 2018 14:57:19 +0000 Subject: [PATCH 03/49] Support mounting /sysroot (and /boot) read-only We want to support extending the read-only state to cover `/sysroot` and `/boot`, since conceptually all of the data there should only be written via libostree. Or at least for `/boot` should *mostly* just be written by ostree. This change needs to be opt-in though to avoid breaking anyone. Add a `sysroot/readonly` key to the repository config which instructs `ostree-remount.service` to ensure `/sysroot` is read-only. This requires a bit of a dance because `/sysroot` is actually the same filesystem as `/`; so we make `/etc` a writable bind mount in this case. We also need to handle `/var` in the "OSTree default" case of a bind mount; the systemd generator now looks at the writability state of `/sysroot` and uses that to determine whether it should have the `var.mount` unit happen before or after `ostree-remount.service.` Also add an API to instruct the libostree shared library that the caller has created a new mount namespace. This way we can freely remount read-write. This approach extends upon in a much better way previous work we did to support remounting `/boot` read-write. Closes: https://github.com/ostreedev/ostree/issues/1265 --- Makefile-switchroot.am | 3 +- apidoc/ostree-sections.txt | 3 + src/boot/ostree-remount.service | 6 +- src/libostree/libostree-devel.sym | 3 + src/libostree/ostree-impl-system-generator.c | 3 +- src/libostree/ostree-sysroot-cleanup.c | 8 +- src/libostree/ostree-sysroot-deploy.c | 29 ++- src/libostree/ostree-sysroot-private.h | 13 +- src/libostree/ostree-sysroot.c | 200 +++++++++++++++---- src/libostree/ostree-sysroot.h | 14 ++ src/ostree/ot-main.c | 37 ++++ src/switchroot/ostree-remount.c | 78 +++++++- 12 files changed, 330 insertions(+), 67 deletions(-) diff --git a/Makefile-switchroot.am b/Makefile-switchroot.am index b81b843f..ad370eb7 100644 --- a/Makefile-switchroot.am +++ b/Makefile-switchroot.am @@ -55,7 +55,8 @@ ostree_remount_SOURCES = \ src/switchroot/ostree-mount-util.h \ src/switchroot/ostree-remount.c \ $(NULL) -ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) -Isrc/switchroot +ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -Isrc/switchroot -I$(srcdir)/libglnx +ostree_remount_LDADD = $(AM_LDFLAGS) $(OT_INTERNAL_GIO_UNIX_LIBS) libglnx.la if BUILDOPT_SYSTEMD ostree_prepare_root_CPPFLAGS += -DHAVE_SYSTEMD=1 diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index f99c4df5..1ef4bbf6 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -499,6 +499,7 @@ ostree_sepolicy_get_type OstreeSysroot ostree_sysroot_new ostree_sysroot_new_default +ostree_sysroot_initialize ostree_sysroot_get_path ostree_sysroot_load ostree_sysroot_load_if_changed @@ -508,6 +509,8 @@ ostree_sysroot_lock_async ostree_sysroot_lock_finish ostree_sysroot_unlock ostree_sysroot_unload +ostree_sysroot_set_mount_namespace_in_use +ostree_sysroot_is_booted ostree_sysroot_get_fd ostree_sysroot_ensure_initialized ostree_sysroot_get_bootversion diff --git a/src/boot/ostree-remount.service b/src/boot/ostree-remount.service index b98110c2..4c3ed94a 100644 --- a/src/boot/ostree-remount.service +++ b/src/boot/ostree-remount.service @@ -22,12 +22,12 @@ DefaultDependencies=no ConditionKernelCommandLine=ostree OnFailure=emergency.target Conflicts=umount.target -After=-.mount +# Run after core mounts +After=-.mount var.mount After=systemd-remount-fs.service +# But we run *before* most other core bootup services that need write access to /etc and /var Before=local-fs.target umount.target -# Other early boot units that need to write to /var Before=systemd-random-seed.service plymouth-read-write.service systemd-journal-flush.service -# tmpfiles.d usually needs write access to a few places Before=systemd-tmpfiles-setup.service [Service] diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index a5d15e93..d1666176 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,9 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { + ostree_sysroot_initialize; + ostree_sysroot_is_booted; + ostree_sysroot_set_mount_namespace_in_use; } LIBOSTREE_2019.6; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c index ce40a698..cb9170b3 100644 --- a/src/libostree/ostree-impl-system-generator.c +++ b/src/libostree/ostree-impl-system-generator.c @@ -28,6 +28,7 @@ #ifdef HAVE_LIBMOUNT #include #endif +#include #include #include "otutil.h" @@ -189,8 +190,6 @@ _ostree_impl_system_generator (const char *ostree_cmdline, "[Unit]\n" "Documentation=man:ostree(1)\n" "ConditionKernelCommandLine=!systemd.volatile\n" - /* We need /sysroot mounted writable first */ - "After=ostree-remount.service\n" "Before=local-fs.target\n" "\n" "[Mount]\n" diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index ef95d13c..71d978e0 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -455,6 +455,9 @@ ostree_sysroot_cleanup_prune_repo (OstreeSysroot *sysroot, OstreeRepo *repo = ostree_sysroot_repo (sysroot); const guint depth = 0; /* Historical default */ + if (!_ostree_sysroot_ensure_writable (sysroot, error)) + return FALSE; + /* Hold an exclusive lock by default across gathering refs and doing * the prune. */ @@ -535,7 +538,10 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, GError **error) { g_return_val_if_fail (OSTREE_IS_SYSROOT (self), FALSE); - g_return_val_if_fail (self->loaded, FALSE); + g_return_val_if_fail (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED, FALSE); + + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; if (!cleanup_other_bootversions (self, cancellable, error)) return glnx_prefix_error (error, "Cleaning bootversions"); diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index a09c354b..0dcda732 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -56,6 +56,9 @@ #define OSTREE_DEPLOYMENT_FINALIZING_ID SD_ID128_MAKE(e8,64,6c,d6,3d,ff,46,25,b7,79,09,a8,e7,a4,09,94) #endif +static gboolean +is_ro_mount (const char *path); + /* * Like symlinkat() but overwrites (atomically) an existing * symlink. @@ -806,6 +809,9 @@ write_origin_file_internal (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { + if (!_ostree_sysroot_ensure_writable (sysroot, error)) + return FALSE; + GLNX_AUTO_PREFIX_ERROR ("Writing out origin file", error); GKeyFile *origin = new_origin ? new_origin : ostree_deployment_get_origin (deployment); @@ -2217,7 +2223,10 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - g_assert (self->loaded); + g_assert (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED); + + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; /* Dealing with the staged deployment is quite tricky here. This function is * primarily concerned with writing out "finalized" deployments which have @@ -2374,7 +2383,6 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, if (boot_was_ro_mount) { - /* TODO: Use new mount namespace. https://github.com/ostreedev/ostree/issues/1265 */ if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) return glnx_throw_errno_prefix (error, "Remounting /boot read-write"); } @@ -2408,8 +2416,10 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, /* Note equivalent of try/finally here */ gboolean success = write_deployments_bootswap (self, new_deployments, opts, bootloader, &syncstats, cancellable, error); - /* Below here don't set GError until the if (!success) check */ - if (boot_was_ro_mount) + /* Below here don't set GError until the if (!success) check. + * Note we only bother remounting if a mount namespace isn't in use. + * */ + if (boot_was_ro_mount && !self->mount_namespace_in_use) { if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_RDONLY | MS_SILENT, NULL) < 0) { @@ -2716,6 +2726,9 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + g_autoptr(OstreeDeployment) deployment = NULL; if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv, &deployment, cancellable, error)) @@ -2817,6 +2830,9 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (self); if (booted_deployment == NULL) return glnx_throw (error, "Cannot stage a deployment when not currently booted into an OSTree system"); @@ -3043,6 +3059,9 @@ ostree_sysroot_deployment_set_kargs (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + /* For now; instead of this do a redeployment */ g_assert (!ostree_deployment_is_staged (deployment)); @@ -3090,6 +3109,8 @@ ostree_sysroot_deployment_set_mutable (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 858673c5..96670b13 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -40,6 +40,12 @@ typedef enum { OSTREE_SYSROOT_DEBUG_TEST_STAGED_PATH = 1 << 3, } OstreeSysrootDebugFlags; +typedef enum { + OSTREE_SYSROOT_LOAD_STATE_NONE, /* ostree_sysroot_new() was called */ + OSTREE_SYSROOT_LOAD_STATE_INIT, /* We've loaded basic sysroot state and have an fd */ + OSTREE_SYSROOT_LOAD_STATE_LOADED, /* We've loaded all of the deployments */ +} OstreeSysrootLoadState; + /** * OstreeSysroot: * Internal struct @@ -51,7 +57,8 @@ struct OstreeSysroot { int sysroot_fd; GLnxLockFile lock; - gboolean loaded; + OstreeSysrootLoadState loadstate; + gboolean mount_namespace_in_use; /* TRUE if caller has told us they used CLONE_NEWNS */ gboolean root_is_ostree_booted; /* TRUE if sysroot is / and we are booted via ostree */ /* The device/inode for /, used to detect booted deployment */ dev_t root_device; @@ -79,6 +86,10 @@ struct OstreeSysroot { #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_DIR "/run/ostree/deployment-state/" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT "unlocked-development" +gboolean +_ostree_sysroot_ensure_writable (OstreeSysroot *self, + GError **error); + void _ostree_sysroot_emit_journal_msg (OstreeSysroot *self, const char *msg); diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 1c9dbf37..23a06975 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -226,6 +226,33 @@ ostree_sysroot_new_default (void) return ostree_sysroot_new (NULL); } +/** + * ostree_sysroot_set_mount_namespace_in_use: + * + * If this function is invoked, then libostree will assume that + * a private Linux mount namespace has been created by the process. + * The primary use case for this is to have e.g. /sysroot mounted + * read-only by default. + * + * If this function has been called, then when a function which requires + * writable access is invoked, libostree will automatically remount as writable + * any mount points on which it operates. This currently is just `/sysroot` and + * `/boot`. + * + * If you invoke this function, it must be before ostree_sysroot_load(); it may + * be invoked before or after ostree_sysroot_initialize(). + * + * Since: 2019.7 + */ +void +ostree_sysroot_set_mount_namespace_in_use (OstreeSysroot *self) +{ + /* Must be before we're loaded, as otherwise we'd have to close/reopen all our + fds, e.g. the repo */ + g_return_if_fail (self->loadstate < OSTREE_SYSROOT_LOAD_STATE_LOADED); + self->mount_namespace_in_use = TRUE; +} + /** * ostree_sysroot_get_path: * @self: @@ -238,6 +265,7 @@ ostree_sysroot_get_path (OstreeSysroot *self) return self->path; } +/* Open a directory file descriptor for the sysroot if we haven't yet */ static gboolean ensure_sysroot_fd (OstreeSysroot *self, GError **error) @@ -251,13 +279,51 @@ ensure_sysroot_fd (OstreeSysroot *self, return TRUE; } +/* Remount /sysroot read-write if necessary */ +gboolean +_ostree_sysroot_ensure_writable (OstreeSysroot *self, + GError **error) +{ + /* Do nothing if no mount namespace is in use */ + if (!self->mount_namespace_in_use) + return TRUE; + + /* If a mount namespace is in use, ensure we're initialized */ + if (!ostree_sysroot_initialize (self, error)) + return FALSE; + + /* If we aren't operating on a booted system, then we don't + * do anything with mounts. + */ + if (!self->root_is_ostree_booted) + return TRUE; + + /* Check if /sysroot is a read-only mountpoint */ + struct statvfs stvfsbuf; + if (statvfs ("/sysroot", &stvfsbuf) < 0) + return glnx_throw_errno_prefix (error, "fstatvfs(/sysroot)"); + if ((stvfsbuf.f_flag & ST_RDONLY) == 0) + return TRUE; + + /* OK, let's remount writable. */ + if (mount ("/sysroot", "/sysroot", NULL, MS_REMOUNT | MS_RELATIME, "") < 0) + return glnx_throw_errno_prefix (error, "Remounting /sysroot read-write"); + + /* Reopen our fd */ + glnx_close_fd (&self->sysroot_fd); + if (!ensure_sysroot_fd (self, error)) + return FALSE; + + return TRUE; +} + /** * ostree_sysroot_get_fd: * @self: Sysroot * - * Access a file descriptor that refers to the root directory of this - * sysroot. ostree_sysroot_load() must have been invoked prior to - * calling this function. + * Access a file descriptor that refers to the root directory of this sysroot. + * ostree_sysroot_initialize() (or ostree_sysroot_load()) must have been invoked + * prior to calling this function. * * Returns: A file descriptor valid for the lifetime of @self */ @@ -268,6 +334,22 @@ ostree_sysroot_get_fd (OstreeSysroot *self) return self->sysroot_fd; } +/** + * ostree_sysroot_is_booted: + * @self: Sysroot + * + * Can only be invoked after `ostree_sysroot_initialize()`. + * + * Returns: %TRUE iff the sysroot points to a booted deployment + * Since: 2019.7 + */ +gboolean +ostree_sysroot_is_booted (OstreeSysroot *self) +{ + g_return_val_if_fail (self->loadstate >= OSTREE_SYSROOT_LOAD_STATE_INIT, FALSE); + return self->root_is_ostree_booted; +} + gboolean _ostree_sysroot_bump_mtime (OstreeSysroot *self, GError **error) @@ -798,6 +880,57 @@ ensure_repo (OstreeSysroot *self, return TRUE; } +/** + * ostree_sysroot_initialize: + * @self: sysroot + * + * Subset of ostree_sysroot_load(); performs basic initialization. Notably, one + * can invoke `ostree_sysroot_get_fd()` after calling this function. + * + * It is not necessary to call this function if ostree_sysroot_load() is + * invoked. + * + * Since: 2019.7 + */ +gboolean +ostree_sysroot_initialize (OstreeSysroot *self, + GError **error) +{ + if (!ensure_sysroot_fd (self, error)) + return FALSE; + + if (self->loadstate < OSTREE_SYSROOT_LOAD_STATE_INIT) + { + /* Gather some global state; first if we have the global ostree-booted flag; + * we'll use it to sanity check that we found a booted deployment for example. + * Second, we also find out whether sysroot == /. + */ + if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) + return FALSE; + const gboolean ostree_booted = (errno == 0); + + { struct stat root_stbuf; + if (!glnx_fstatat (AT_FDCWD, "/", &root_stbuf, 0, error)) + return FALSE; + self->root_device = root_stbuf.st_dev; + self->root_inode = root_stbuf.st_ino; + } + + struct stat self_stbuf; + if (!glnx_fstatat (AT_FDCWD, gs_file_get_path_cached (self->path), &self_stbuf, 0, error)) + return FALSE; + + const gboolean root_is_sysroot = + (self->root_device == self_stbuf.st_dev && + self->root_inode == self_stbuf.st_ino); + + self->root_is_ostree_booted = (ostree_booted && root_is_sysroot); + self->loadstate = OSTREE_SYSROOT_LOAD_STATE_INIT; + } + + return TRUE; +} + /* Reload the staged deployment from the file in /run */ gboolean _ostree_sysroot_reload_staged (OstreeSysroot *self, @@ -870,7 +1003,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - if (!ensure_sysroot_fd (self, error)) + if (!ostree_sysroot_initialize (self, error)) return FALSE; /* Here we also lazily initialize the repository. We didn't do this @@ -880,34 +1013,6 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!ensure_repo (self, error)) return FALSE; - /* Gather some global state; first if we have the global ostree-booted flag; - * we'll use it to sanity check that we found a booted deployment for example. - * Second, we also find out whether sysroot == /. - */ - if (!self->loaded) - { - if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) - return FALSE; - const gboolean ostree_booted = (errno == 0); - - { struct stat root_stbuf; - if (!glnx_fstatat (AT_FDCWD, "/", &root_stbuf, 0, error)) - return FALSE; - self->root_device = root_stbuf.st_dev; - self->root_inode = root_stbuf.st_ino; - } - - struct stat self_stbuf; - if (!glnx_fstat (self->sysroot_fd, &self_stbuf, error)) - return FALSE; - - const gboolean root_is_sysroot = - (self->root_device == self_stbuf.st_dev && - self->root_inode == self_stbuf.st_ino); - - self->root_is_ostree_booted = (ostree_booted && root_is_sysroot); - } - int bootversion = 0; if (!read_current_bootversion (self, &bootversion, cancellable, error)) return FALSE; @@ -990,8 +1095,8 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, ostree_deployment_set_index (deployment, i); } - /* Determine whether we're "physical" or not, the first time we initialize */ - if (!self->loaded) + /* Determine whether we're "physical" or not, the first time we load deployments */ + if (self->loadstate < OSTREE_SYSROOT_LOAD_STATE_LOADED) { /* If we have a booted deployment, the sysroot is / and we're definitely * not physical. @@ -1009,13 +1114,14 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, self->is_physical = TRUE; } /* Otherwise, the default is FALSE */ + + self->loadstate = OSTREE_SYSROOT_LOAD_STATE_LOADED; } self->bootversion = bootversion; self->subbootversion = subbootversion; self->deployments = deployments; deployments = NULL; /* Transfer ownership */ - self->loaded = TRUE; self->loaded_ts = stbuf.st_mtim; if (out_changed) @@ -1044,7 +1150,7 @@ ostree_sysroot_get_subbootversion (OstreeSysroot *self) OstreeDeployment * ostree_sysroot_get_booted_deployment (OstreeSysroot *self) { - g_return_val_if_fail (self->loaded, NULL); + g_return_val_if_fail (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED, NULL); return self->booted_deployment; } @@ -1060,7 +1166,7 @@ ostree_sysroot_get_booted_deployment (OstreeSysroot *self) OstreeDeployment * ostree_sysroot_get_staged_deployment (OstreeSysroot *self) { - g_return_val_if_fail (self->loaded, NULL); + g_return_val_if_fail (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED, NULL); return self->staged_deployment; } @@ -1074,7 +1180,7 @@ ostree_sysroot_get_staged_deployment (OstreeSysroot *self) GPtrArray * ostree_sysroot_get_deployments (OstreeSysroot *self) { - g_return_val_if_fail (self->loaded, NULL); + g_return_val_if_fail (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED, NULL); GPtrArray *copy = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); for (guint i = 0; i < self->deployments->len; i++) @@ -1163,8 +1269,8 @@ ostree_sysroot_get_repo (OstreeSysroot *self, * @self: Sysroot * * This function is a variant of ostree_sysroot_get_repo() that cannot fail, and - * returns a cached repository. Can only be called after ostree_sysroot_load() - * has been invoked successfully. + * returns a cached repository. Can only be called after ostree_sysroot_initialize() + * or ostree_sysroot_load() has been invoked successfully. * * Returns: (transfer none): The OSTree repository in sysroot @self. * @@ -1173,7 +1279,7 @@ ostree_sysroot_get_repo (OstreeSysroot *self, OstreeRepo * ostree_sysroot_repo (OstreeSysroot *self) { - g_return_val_if_fail (self->loaded, NULL); + g_return_val_if_fail (self->loadstate >= OSTREE_SYSROOT_LOAD_STATE_LOADED, NULL); g_assert (self->repo); return self->repo; } @@ -1368,6 +1474,10 @@ ostree_sysroot_lock (OstreeSysroot *self, { if (!ensure_sysroot_fd (self, error)) return FALSE; + + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + return glnx_make_lock_file (self->sysroot_fd, OSTREE_SYSROOT_LOCKFILE, LOCK_EX, &self->lock, error); } @@ -1391,12 +1501,14 @@ ostree_sysroot_try_lock (OstreeSysroot *self, gboolean *out_acquired, GError **error) { - g_autoptr(GError) local_error = NULL; - if (!ensure_sysroot_fd (self, error)) return FALSE; + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + /* Note use of LOCK_NB */ + g_autoptr(GError) local_error = NULL; if (!glnx_make_lock_file (self->sysroot_fd, OSTREE_SYSROOT_LOCKFILE, LOCK_EX | LOCK_NB, &self->lock, &local_error)) { @@ -1509,7 +1621,7 @@ ostree_sysroot_init_osname (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - if (!ensure_sysroot_fd (self, error)) + if (!_ostree_sysroot_ensure_writable (self, error)) return FALSE; const char *deploydir = glnx_strjoina ("ostree/deploy/", osname); diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index 502cd750..af8192fc 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -41,12 +41,22 @@ OstreeSysroot* ostree_sysroot_new (GFile *path); _OSTREE_PUBLIC OstreeSysroot* ostree_sysroot_new_default (void); +_OSTREE_PUBLIC +void ostree_sysroot_set_mount_namespace_in_use (OstreeSysroot *self); + _OSTREE_PUBLIC GFile *ostree_sysroot_get_path (OstreeSysroot *self); +_OSTREE_PUBLIC +gboolean ostree_sysroot_is_booted (OstreeSysroot *self); + _OSTREE_PUBLIC int ostree_sysroot_get_fd (OstreeSysroot *self); +_OSTREE_PUBLIC +gboolean ostree_sysroot_initialize (OstreeSysroot *self, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_load (OstreeSysroot *self, GCancellable *cancellable, @@ -90,6 +100,10 @@ GFile * ostree_sysroot_get_deployment_origin_path (GFile *deployment_path); _OSTREE_PUBLIC gboolean ostree_sysroot_lock (OstreeSysroot *self, GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sysroot_lock_with_mount_namespace (OstreeSysroot *self, GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_try_lock (OstreeSysroot *self, gboolean *out_acquired, diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 4b72f399..a044cef2 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -27,6 +27,7 @@ #include #include +#include #include "ot-main.h" #include "ostree.h" @@ -434,10 +435,46 @@ ostree_admin_option_context_parse (GOptionContext *context, sysroot_path = g_file_new_for_path (opt_sysroot); g_autoptr(OstreeSysroot) sysroot = ostree_sysroot_new (sysroot_path); + if (!ostree_sysroot_initialize (sysroot, error)) + return FALSE; g_signal_connect (sysroot, "journal-msg", G_CALLBACK (on_sysroot_journal_msg), NULL); if ((flags & OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED) == 0) { + /* If we're requested to lock the sysroot, first check if we're operating + * on a booted (not physical) sysroot. Then find out if the /sysroot + * subdir is a read-only mount point, and if so, create a new mount + * namespace and tell the sysroot that we've done so. See the docs for + * ostree_sysroot_set_mount_namespace_in_use(). + * + * This is a conservative approach; we could just always + * unshare() too. + */ + if (ostree_sysroot_is_booted (sysroot)) + { + int sysroot_fd = ostree_sysroot_get_fd (sysroot); + g_assert_cmpint (sysroot_fd, !=, -1); + + glnx_autofd int sysroot_subdir_fd = glnx_opendirat_with_errno (sysroot_fd, "sysroot", TRUE); + if (sysroot_subdir_fd < 0) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "opendirat"); + } + else if (getuid () == 0) + { + struct statvfs stvfs; + if (fstatvfs (sysroot_subdir_fd, &stvfs) < 0) + return glnx_throw_errno_prefix (error, "fstatvfs"); + if (stvfs.f_flag & ST_RDONLY) + { + if (unshare (CLONE_NEWNS) < 0) + return glnx_throw_errno_prefix (error, "preparing writable sysroot: unshare (CLONE_NEWNS)"); + ostree_sysroot_set_mount_namespace_in_use (sysroot); + } + } + } + /* Released when sysroot is finalized, or on process exit */ if (!ot_admin_sysroot_lock (sysroot, error)) return FALSE; diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 5e6d23d3..326b104f 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -37,10 +38,14 @@ #include #include +#include + #include "ostree-mount-util.h" +#include "glnx-backport-autocleanups.h" static void -do_remount (const char *target) +do_remount (const char *target, + bool writable) { struct stat stbuf; if (lstat (target, &stbuf) < 0) @@ -54,20 +59,41 @@ do_remount (const char *target) struct statvfs stvfsbuf; if (statvfs (target, &stvfsbuf) == -1) return; - /* If no read-only flag, skip it */ - if ((stvfsbuf.f_flag & ST_RDONLY) == 0) + + const bool currently_writable = ((stvfsbuf.f_flag & ST_RDONLY) == 0); + if (writable == currently_writable) return; - /* It's a mounted, read-only fs; remount it */ - if (mount (target, target, NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) + + int mnt_flags = MS_REMOUNT | MS_SILENT; + if (!writable) + mnt_flags |= MS_RDONLY; + if (mount (target, target, NULL, mnt_flags, NULL) < 0) { /* Also ignore EINVAL - if the target isn't a mountpoint * already, then assume things are OK. */ - if (errno != EINVAL) - err (EXIT_FAILURE, "failed to remount %s", target); + if (errno != EINVAL) + err (EXIT_FAILURE, "failed to remount(%s) %s", writable ? "rw" : "ro", target); + else + return; } - else - printf ("Remounted: %s\n", target); + + printf ("Remounted %s: %s\n", writable ? "rw" : "ro", target); +} + +static bool +sysroot_is_configured_ro (void) +{ + struct stat stbuf; + static const char config_path[] = "/ostree/repo/config"; + if (stat (config_path, &stbuf) != 0) + return false; + + g_autoptr(GKeyFile) keyfile = g_key_file_new (); + if (!g_key_file_load_from_file (keyfile, config_path, 0, NULL)) + return false; + + return g_key_file_get_boolean (keyfile, "sysroot", "readonly", NULL); } int @@ -95,8 +121,38 @@ main(int argc, char *argv[]) exit (EXIT_SUCCESS); } - do_remount ("/sysroot"); - do_remount ("/var"); + /* Query the repository configuration - this is an operating system builder + * choice. + * */ + const bool sysroot_readonly = sysroot_is_configured_ro (); + + /* Mount the sysroot read-only if we're configured to do so. + * Note we only get here if / is already writable. + */ + do_remount ("/sysroot", !sysroot_readonly); + + if (sysroot_readonly) + { + /* Now, /etc is not normally a bind mount, but remounting the + * sysroot above made it read-only since it's on the same filesystem. + * Make it a self-bind mount, so we can then mount it read-write. + */ + if (mount ("/etc", "/etc", NULL, MS_BIND, NULL) < 0) + err (EXIT_FAILURE, "failed to make /etc a bind mount"); + do_remount ("/etc", true); + } + + /* If /var was created as as an OSTree default bind mount (instead of being a separate filesystem) + * then remounting the root mount read-only also remounted it. + * So just like /etc, we need to make it read-write by default. + * If it was a separate filesystem, we expect it to be writable anyways, + * so it doesn't hurt to remount it if so. + * + * And if we started out with a writable system root, then we need + * to ensure that the /var bind mount created by the systemd generator + * is writable too. + */ + do_remount ("/var", true); exit (EXIT_SUCCESS); } From f6867358e2fb50382b6ff3cbf717a1a733965338 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Sun, 29 Dec 2019 12:32:28 +0000 Subject: [PATCH 04/49] test-switchroot.sh: Exclude /proc from file list Since we're not interested in any file inside /proc, exclude it from the file listing in our fake root thus avoiding failures when processes die during our execution and find(1) can't then look inside those directories. Signed-off-by: Alex Kiernan --- tests/test-switchroot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-switchroot.sh b/tests/test-switchroot.sh index bc3ec38b..b05b11a4 100755 --- a/tests/test-switchroot.sh +++ b/tests/test-switchroot.sh @@ -56,7 +56,7 @@ find_in_env() { "$1" "$tmpdir" enter_fs "$tmpdir" ostree-prepare-root /sysroot - find / + find / \( -path /proc -o -path /sysroot/proc \) -prune -o -print touch /usr/usr_writable 2>/null \ && echo "/usr is writable" \ || echo "/usr is not writable" From 5c62a7e4d0a5614cacc552052d474fe1768583ec Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Sun, 29 Dec 2019 13:23:23 +0000 Subject: [PATCH 05/49] build: Expose systemd in OSTREE_FEATURES Signed-off-by: Alex Kiernan --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 4a752e55..429239f4 100644 --- a/configure.ac +++ b/configure.ac @@ -533,6 +533,7 @@ dnl If we have both, we use the "new /var" model with ostree-system-generator AM_CONDITIONAL(BUILDOPT_SYSTEMD_AND_LIBMOUNT,[test x$with_systemd = xyes && test x$with_libmount = xyes]) AM_COND_IF(BUILDOPT_SYSTEMD_AND_LIBMOUNT, AC_DEFINE([BUILDOPT_LIBSYSTEMD_AND_LIBMOUNT], 1, [Define if systemd and libmount])) +if test x$with_systemd != xno; then OSTREE_FEATURES="$OSTREE_FEATURES systemd"; fi AC_ARG_WITH(builtin-grub2-mkconfig, AS_HELP_STRING([--with-builtin-grub2-mkconfig], From 87ccb400a28dcdcaa07f6c953b32816edb08d875 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Sun, 29 Dec 2019 13:25:24 +0000 Subject: [PATCH 06/49] tests: Skip /var test if running with systemd and libmount If running with systemd and libmount then /var mounting is deferred for systemd. Skip the relevant tests in this case as it will always fail. Signed-off-by: Alex Kiernan --- tests/libtest.sh | 12 ++++++++++++ tests/test-switchroot.sh | 8 ++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 3f5fd931..cbdf331c 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -559,6 +559,18 @@ skip_without_user_xattrs () { fi } +_have_systemd_and_libmount='' +have_systemd_and_libmount() { + if test "${_have_systemd_and_libmount}" = ''; then + if [ $(ostree --version | grep -c -e '- systemd' -e '- libmount') -eq 2 ]; then + _have_systemd_and_libmount=yes + else + _have_systemd_and_libmount=no + fi + fi + test ${_have_systemd_and_libmount} = yes +} + # Skip unless SELinux is disabled, or we can relabel. # Default Docker has security.selinux xattrs, but returns # EOPNOTSUPP when trying to set them, even to the existing value. diff --git a/tests/test-switchroot.sh b/tests/test-switchroot.sh index b05b11a4..e66c68de 100755 --- a/tests/test-switchroot.sh +++ b/tests/test-switchroot.sh @@ -82,7 +82,9 @@ test_that_prepare_root_sets_sysroot_up_correctly_with_initrd() { grep -qx "/this_is_bootfs" files grep -qx "/sysroot/this_is_ostree_root" files grep -qx "/sysroot/sysroot/this_is_real_root" files - grep -qx "/sysroot/var/this_is_ostree_var" files + if ! have_systemd_and_libmount; then + grep -qx "/sysroot/var/this_is_ostree_var" files + fi grep -qx "/sysroot/usr/this_is_ostree_usr" files grep -qx "/sysroot/usr is not writable" files @@ -101,7 +103,9 @@ test_that_prepare_root_sets_root_up_correctly_with_no_initrd() { grep -qx "/this_is_ostree_root" files grep -qx "/sysroot/this_is_bootfs" files grep -qx "/sysroot/this_is_real_root" files - grep -qx "/var/this_is_ostree_var" files + if ! have_systemd_and_libmount; then + grep -qx "/var/this_is_ostree_var" files + fi grep -qx "/usr/this_is_ostree_usr" files grep -qx "/usr is not writable" files From e4db245bec66f353c6aba1a661465ce649bd521e Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Sun, 29 Dec 2019 12:43:34 +0000 Subject: [PATCH 07/49] test-switchroot.sh: Find ostree-prepare-root in installed tests When running with installed tests, ostree-prepare-root (probably) exists in /usr/lib. Add heuristics to look for it based on the directory we're running from. Signed-off-by: Alex Kiernan --- tests/test-switchroot.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test-switchroot.sh b/tests/test-switchroot.sh index bc3ec38b..2adaa0ac 100755 --- a/tests/test-switchroot.sh +++ b/tests/test-switchroot.sh @@ -2,6 +2,16 @@ this_script="${BASH_SOURCE:-$(readlink -f "$0")}" +OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../ostree-prepare-root +if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then + # ostree-prepare-root is in $libdir by default, assume we can find it + # based on our test directory, if not we'll have to skip this test. + OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../../../lib/ostree/ostree-prepare-root + if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then + OSTREE_PREPARE_ROOT="" + fi +fi + setup_bootfs() { mkdir -p "$1/proc" "$1/bin" @@ -13,7 +23,7 @@ setup_bootfs() { mount --bind "$1/override_cmdline" "$1/proc/cmdline" touch "$1/this_is_bootfs" - cp "$(dirname "$this_script")/../ostree-prepare-root" "$1/bin" + cp "${OSTREE_PREPARE_ROOT}" "$1/bin" } setup_rootfs() { @@ -130,6 +140,9 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then [ -f /bin/busybox ] || \ skip "this test needs busybox" + [ -n "${OSTREE_PREPARE_ROOT}" ] || \ + skip "this test needs ostree-prepare-root" + echo "1..3" test_that_prepare_root_sets_sysroot_up_correctly_with_initrd test_that_prepare_root_sets_root_up_correctly_with_no_initrd From d61183ce43bcec2fc943f9da7429968d97e237e9 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Mon, 30 Dec 2019 13:28:35 +0000 Subject: [PATCH 08/49] fixup! test-switchroot.sh: Find ostree-prepare-root in installed tests --- tests/test-switchroot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-switchroot.sh b/tests/test-switchroot.sh index 2adaa0ac..c472fe54 100755 --- a/tests/test-switchroot.sh +++ b/tests/test-switchroot.sh @@ -140,7 +140,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then [ -f /bin/busybox ] || \ skip "this test needs busybox" - [ -n "${OSTREE_PREPARE_ROOT}" ] || \ + [ -n "${OSTREE_PREPARE_ROOT}" ] || \ skip "this test needs ostree-prepare-root" echo "1..3" From 37045b4b468460785926d73cc2bc4d288caaa9b3 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 13:05:48 -0600 Subject: [PATCH 09/49] lib/commit: Only set generate_sizes for archive repos Rather than checking throughout the code, only set the boolean when appropriate. --- src/libostree/ostree-repo-commit.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8c5d9411..8f059e11 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -345,6 +345,19 @@ content_size_cache_entry_free (gpointer entry) g_slice_free (OstreeContentSizeCacheEntry, entry); } +static void +repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier) +{ + if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) + { + if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) + self->generate_sizes = TRUE; + else + g_debug ("Not generating sizes for non-archive repo"); + } +} + static void repo_store_size_entry (OstreeRepo *self, const gchar *checksum, @@ -956,7 +969,6 @@ write_content_object (OstreeRepo *self, g_auto(OtCleanupUnlinkat) tmp_unlinker = { commit_tmp_dfd (self), NULL }; g_auto(GLnxTmpfile) tmpf = { 0, }; goffset unpacked_size = 0; - gboolean indexable = FALSE; /* Is it a symlink physically? */ if (phys_object_is_symlink) { @@ -982,9 +994,6 @@ write_content_object (OstreeRepo *self, g_assert (repo_mode == OSTREE_REPO_MODE_ARCHIVE); - if (self->generate_sizes) - indexable = TRUE; - if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, &tmpf, error)) return FALSE; @@ -1108,7 +1117,7 @@ write_content_object (OstreeRepo *self, else { /* Update size metadata if configured */ - if (indexable && object_file_type == G_FILE_TYPE_REGULAR) + if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR) { struct stat stbuf; @@ -3848,8 +3857,7 @@ ostree_repo_write_directory_to_mtree (OstreeRepo *self, } else { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + repo_setup_generate_sizes (self, modifier); g_autoptr(GPtrArray) path = g_ptr_array_new (); if (!write_directory_to_mtree_internal (self, dir, mtree, modifier, path, @@ -3883,8 +3891,7 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) - self->generate_sizes = TRUE; + repo_setup_generate_sizes (self, modifier); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error)) From 694b741a366f4abb523f6e4ffbec3f56c5934d1a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 22 Oct 2019 14:59:19 -0600 Subject: [PATCH 10/49] tests/sizes: Improve metadata validation Ensure all 3 of the checksum, compressed size and uncompressed size are correct. For repeatable objects, skip xattrs and use canonical permissions for the commit. For the sizes, read a varint rather than assuming they will be a single byte. To work around bugs in gjs with byte array unpacking, manually build the array byte by byte. Split out some helper functions to use in subsequent tests. --- tests/test-sizes.js | 126 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 27 deletions(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 73b179c5..622c2d1b 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -28,6 +28,96 @@ function assertEquals(a, b) { throw new Error("assertion failed " + JSON.stringify(a) + " == " + JSON.stringify(b)); } +function assertGreater(a, b) { + if (a <= b) + throw new Error("assertion failed " + JSON.stringify(a) + " > " + JSON.stringify(b)); +} + +function assertGreaterEquals(a, b) { + if (a < b) + throw new Error("assertion failed " + JSON.stringify(a) + " >= " + JSON.stringify(b)); +} + +// Adapted from _ostree_read_varuint64() +function readVarint(buffer) { + let result = 0; + let count = 0; + let len = buffer.length; + let cur; + + do { + assertGreater(len, 0); + cur = buffer[count]; + result = result | ((cur & 0x7F) << (7 * count)); + count++; + len--; + } while (cur & 0x80); + + return [result, count]; +} + +// There have been various bugs with byte array unpacking in GJS, so +// just do it manually. +function unpackByteArray(variant) { + let array = []; + let nBytes = variant.n_children(); + for (let i = 0; i < nBytes; i++) { + array.push(variant.get_child_value(i).get_byte()); + } + return array; +} + +function validateSizes(repo, commit, expectedFiles) { + let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); + let metadata = commitVariant.get_child_value(0); + let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); + let nSizes = sizes.n_children(); + let expectedNSizes = Object.keys(expectedFiles).length + assertEquals(nSizes, expectedNSizes); + + for (let i = 0; i < nSizes; i++) { + let sizeEntry = sizes.get_child_value(i); + assertGreaterEquals(sizeEntry.n_children(), 34); + let entryBytes = unpackByteArray(sizeEntry); + let checksumBytes = entryBytes.slice(0, 32); + let checksumString = OSTree.checksum_from_bytes(checksumBytes); + print("checksum = " + checksumString); + + // Read the sizes from the next 2 varints + let remainingBytes = entryBytes.slice(32); + assertGreaterEquals(remainingBytes.length, 2); + let varintRead; + let compressedSize; + let uncompressedSize; + [compressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertGreaterEquals(remainingBytes.length, 1); + [uncompressedSize, varintRead] = readVarint(remainingBytes); + remainingBytes = remainingBytes.slice(varintRead); + assertEquals(remainingBytes.length, 0); + print("compressed = " + compressedSize); + print("uncompressed = " + uncompressedSize); + + if (!(checksumString in expectedFiles)) { + throw new Error("Checksum " + checksumString + " not in " + + JSON.stringify(expectedFiles)); + } + let expectedSizes = expectedFiles[checksumString]; + let expectedCompressedSize = expectedSizes[0]; + let expectedUncompressedSize = expectedSizes[1]; + if (compressedSize != expectedCompressedSize) { + throw new Error("Compressed size " + compressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedCompressedSize); + } + if (uncompressedSize != expectedUncompressedSize) { + throw new Error("Uncompressed size " + uncompressedSize + + " for checksum " + checksumString + + " does not match expected " + expectedUncompressedSize); + } + } +} + print('1..1') let testDataDir = Gio.File.new_for_path('test-data'); @@ -41,7 +131,10 @@ repo.create(OSTree.RepoMode.ARCHIVE_Z2, null); repo.open(null); -let commitModifier = OSTree.RepoCommitModifier.new(OSTree.RepoCommitModifierFlags.GENERATE_SIZES, null); +let commitModifierFlags = (OSTree.RepoCommitModifierFlags.GENERATE_SIZES | + OSTree.RepoCommitModifierFlags.SKIP_XATTRS | + OSTree.RepoCommitModifierFlags.CANONICAL_PERMISSIONS); +let commitModifier = OSTree.RepoCommitModifier.new(commitModifierFlags, null); assertEquals(repo.get_mode(), OSTree.RepoMode.ARCHIVE_Z2); @@ -56,31 +149,10 @@ print("commit => " + commit); repo.commit_transaction(null); // Test the sizes metadata -let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); -let metadata = commitVariant.get_child_value(0); -let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); -let nSizes = sizes.n_children(); -assertEquals(nSizes, 2); -let expectedUncompressedSizes = [12, 18]; -let foundExpectedUncompressedSizes = 0; -for (let i = 0; i < nSizes; i++) { - let sizeEntry = sizes.get_child_value(i); - assertEquals(sizeEntry.n_children(), 34); - let compressedSize = sizeEntry.get_child_value(32).get_byte(); - let uncompressedSize = sizeEntry.get_child_value(33).get_byte(); - print("compressed = " + compressedSize); - print("uncompressed = " + uncompressedSize); - for (let j = 0; j < expectedUncompressedSizes.length; j++) { - let expected = expectedUncompressedSizes[j]; - if (expected == uncompressedSize) { - print("Matched expected uncompressed size " + expected); - expectedUncompressedSizes.splice(j, 1); - break; - } - } -} -if (expectedUncompressedSizes.length > 0) { - throw new Error("Failed to match expectedUncompressedSizes: " + JSON.stringify(expectedUncompressedSizes)); -} +let expectedFiles = { + 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], + 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], +}; +validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); From 8ec7d6322fe004a93a1b391c279020c5411996c0 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:12:08 -0600 Subject: [PATCH 11/49] lib/commit: Fix object sizes metadata for multiple commits The object sizes hash table was only being cleared when the repo was finalized. That means that performing multiple commits while the repo was open would reuse all the object sizes metadata for each commit. Clear the hash table when the sizes metadata is setup and when it's added to a commit. This still does not fix the issue all the way since it does nothing to prevent the program from constructing multiple commits simultaneously. To handle that, the object sizes hash table should be attached to the MutableTree since that has the commit state. However, the MutableTree is gone when the commit is actually created. The hash table would have to be transferred to the root file when writing the MutableTree. That would be an awkward addition to OstreeRepoFile, though. Add a FIXME to capture that. --- src/libostree/ostree-repo-commit.c | 11 ++++++++++- src/libostree/ostree-repo-private.h | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8f059e11..752a01be 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -352,7 +352,13 @@ repo_setup_generate_sizes (OstreeRepo *self, if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) { if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE) - self->generate_sizes = TRUE; + { + self->generate_sizes = TRUE; + + /* Clear any stale data in the object sizes hash table */ + if (self->object_sizes != NULL) + g_hash_table_remove_all (self->object_sizes); + } else g_debug ("Not generating sizes for non-archive repo"); } @@ -428,6 +434,9 @@ add_size_index_to_metadata (OstreeRepo *self, g_variant_builder_add (builder, "{sv}", "ostree.sizes", g_variant_builder_end (&index_builder)); + + /* Clear the object sizes hash table for a subsequent commit. */ + g_hash_table_remove_all (self->object_sizes); } return g_variant_ref_sink (g_variant_builder_end (builder)); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b57ad799..bc2325e5 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -143,6 +143,14 @@ struct OstreeRepo { guint zlib_compression_level; GHashTable *loose_object_devino_hash; GHashTable *updated_uncompressed_dirs; + + /* FIXME: The object sizes hash table is really per-commit state, not repo + * state. Using a single table for the repo means that commits cannot be + * built simultaneously if they're adding size information. This data should + * probably be in OstreeMutableTree, but that's gone by the time the actual + * commit is constructed. At that point the only commit state is in the root + * OstreeRepoFile. + */ GHashTable *object_sizes; /* Cache the repo's device/inode to use for comparisons elsewhere */ From 44fb5e72a1496d42ff500926a99dbe4c6cb44da6 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:28:11 -0600 Subject: [PATCH 12/49] lib/commit: Make size entries for existing objects If the object was already in the repo then the sizes metadata entry was skipped. Move the sizes entry creation after the data has been computed but before the early return for an existing object. --- src/libostree/ostree-repo-commit.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 752a01be..d995686b 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1060,6 +1060,19 @@ write_content_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ + /* Update size metadata if configured and entry missing */ + if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR && + (self->object_sizes == NULL || + g_hash_table_lookup (self->object_sizes, actual_checksum) == NULL)) + { + struct stat stbuf; + + if (!glnx_fstat (tmpf.fd, &stbuf, error)) + return FALSE; + + repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); + } + /* See whether or not we have the object, now that we know the * checksum. */ @@ -1125,17 +1138,6 @@ write_content_object (OstreeRepo *self, } else { - /* Update size metadata if configured */ - if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR) - { - struct stat stbuf; - - if (!glnx_fstat (tmpf.fd, &stbuf, error)) - return FALSE; - - repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); - } - /* Check if a file with the same payload is present in the repository, and in case try to reflink it */ if (actual_payload_checksum && !_try_clone_from_payload_link (self, self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) @@ -2638,8 +2640,11 @@ ostree_repo_write_content (OstreeRepo *self, { /* First, if we have an expected checksum, see if we already have this * object. This mirrors the same logic in ostree_repo_write_metadata(). + * + * If size metadata is needed, fall through to write_content_object() + * where the entries are made. */ - if (expected_checksum) + if (expected_checksum && !self->generate_sizes) { gboolean have_obj; if (!_ostree_repo_has_loose_object (self, expected_checksum, From 1ea719b76bfa94181eb5d228c5c6be10e14b64f1 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 22 Oct 2019 15:14:58 -0600 Subject: [PATCH 13/49] tests/sizes: Test sizes metadata with existing objects Repeat the commit to make sure that the files are enumerated again for the size metadata. --- tests/test-sizes.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 622c2d1b..a3928fe8 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -118,7 +118,7 @@ function validateSizes(repo, commit, expectedFiles) { } } -print('1..1') +print('1..2') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); @@ -156,3 +156,17 @@ let expectedFiles = { validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); + +// Repeat the commit now that all the objects are cached and ensure the +// metadata is still correct +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Another subject', 'Another body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedFiles); + +print("ok test-sizes repeated"); From 4f1b991246dbc67ab7176842a61f3022bed5aad5 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:10:06 -0600 Subject: [PATCH 14/49] tests/sizes: Test that sizes metadata is not reused Ensure that the object sizes hash table is cleared after a commit and not only when the repo is closed. --- tests/test-sizes.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test-sizes.js b/tests/test-sizes.js index a3928fe8..46848905 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -118,7 +118,7 @@ function validateSizes(repo, commit, expectedFiles) { } } -print('1..2') +print('1..3') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); @@ -157,6 +157,23 @@ validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); +// Remove a file to make sure that metadata is not reused from the +// previous commit +testDataDir.get_child('another-file').delete(null); +delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; + +repo.prepare_transaction(null); +mtree = OSTree.MutableTree.new(); +repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); +[,dirTree] = repo.write_mtree(mtree, null); +[,commit] = repo.write_commit(null, 'Some subject', 'Some body', null, dirTree, null); +print("commit => " + commit); +repo.commit_transaction(null); + +validateSizes(repo, commit, expectedFiles); + +print("ok test-sizes file deleted"); + // Repeat the commit now that all the objects are cached and ensure the // metadata is still correct repo.prepare_transaction(null); From a4592678aa5b71051daed24d08bcc84d8844781c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 23 Oct 2019 09:43:10 -0600 Subject: [PATCH 15/49] tests/sizes: Check duplicate file doesn't add sizes entry A duplicate file will resolve to the same object, so it shouldn't add any entries to the sizes metadata. --- src/libostree/ostree-repo-commit.c | 64 ++++++++++++++++++++++++++---- tests/test-sizes.js | 12 +++++- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index d995686b..f88e2d78 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -365,15 +365,38 @@ repo_setup_generate_sizes (OstreeRepo *self, } static void -repo_store_size_entry (OstreeRepo *self, - const gchar *checksum, - goffset unpacked, - goffset archived) +repo_ensure_size_entries (OstreeRepo *self) { if (G_UNLIKELY (self->object_sizes == NULL)) self->object_sizes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, content_size_cache_entry_free); +} +static gboolean +repo_has_size_entry (OstreeRepo *self, + OstreeObjectType objtype, + const gchar *checksum) +{ + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return TRUE; + + repo_ensure_size_entries (self); + return (g_hash_table_lookup (self->object_sizes, checksum) != NULL); +} + +static void +repo_store_size_entry (OstreeRepo *self, + OstreeObjectType objtype, + const gchar *checksum, + goffset unpacked, + goffset archived) +{ + /* Only file, dirtree and dirmeta objects appropriate for size metadata */ + if (objtype > OSTREE_OBJECT_TYPE_DIR_META) + return; + + repo_ensure_size_entries (self); g_hash_table_replace (self->object_sizes, g_strdup (checksum), content_size_cache_entry_new (unpacked, archived)); @@ -1031,6 +1054,11 @@ write_content_object (OstreeRepo *self, unpacked_size = g_file_info_get_size (file_info); } + else + { + /* For a symlink, the size is the length of the target */ + unpacked_size = strlen (g_file_info_get_symlink_target (file_info)); + } if (!g_output_stream_flush (temp_out, cancellable, error)) return FALSE; @@ -1061,16 +1089,16 @@ write_content_object (OstreeRepo *self, g_assert (actual_checksum != NULL); /* Pacify static analysis */ /* Update size metadata if configured and entry missing */ - if (self->generate_sizes && object_file_type == G_FILE_TYPE_REGULAR && - (self->object_sizes == NULL || - g_hash_table_lookup (self->object_sizes, actual_checksum) == NULL)) + if (self->generate_sizes && + !repo_has_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum)) { struct stat stbuf; if (!glnx_fstat (tmpf.fd, &stbuf, error)) return FALSE; - repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); + repo_store_size_entry (self, OSTREE_OBJECT_TYPE_FILE, actual_checksum, + unpacked_size, stbuf.st_size); } /* See whether or not we have the object, now that we know the @@ -1329,6 +1357,11 @@ write_metadata_object (OstreeRepo *self, */ if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + g_mutex_lock (&self->txn_lock); self->txn.stats.metadata_objects_total++; g_mutex_unlock (&self->txn_lock); @@ -1350,6 +1383,11 @@ write_metadata_object (OstreeRepo *self, gsize len; const guint8 *bufp = g_bytes_get_data (buf, &len); + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, actual_checksum)) + repo_store_size_entry (self, objtype, actual_checksum, len, len); + /* Write the metadata to a temporary file */ g_auto(GLnxTmpfile) tmpf = { 0, }; if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, @@ -2365,6 +2403,16 @@ ostree_repo_write_metadata (OstreeRepo *self, return FALSE; if (have_obj) { + /* Update size metadata if needed */ + if (self->generate_sizes && + !repo_has_size_entry (self, objtype, expected_checksum)) + { + /* Make sure we have a fully serialized object */ + g_autoptr(GVariant) trusted = g_variant_get_normal_form (object); + gsize size = g_variant_get_size (trusted); + repo_store_size_entry (self, objtype, expected_checksum, size, size); + } + if (out_csum) *out_csum = ostree_checksum_to_bytes (expected_checksum); return TRUE; diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 46848905..685fe3fe 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -123,6 +123,10 @@ print('1..3') let testDataDir = Gio.File.new_for_path('test-data'); testDataDir.make_directory(null); testDataDir.get_child('some-file').replace_contents("hello world!", null, false, 0, null); +testDataDir.get_child('some-file').copy(testDataDir.get_child('duplicate-file'), + Gio.FileCopyFlags.OVERWRITE, + null, null); +testDataDir.get_child('link-file').make_symbolic_link('some-file', null); testDataDir.get_child('another-file').replace_contents("hello world again!", null, false, 0, null); let repoPath = Gio.File.new_for_path('repo'); @@ -152,15 +156,21 @@ repo.commit_transaction(null); let expectedFiles = { 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], + '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9], + '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185], + '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12], }; validateSizes(repo, commit, expectedFiles); print("ok test-sizes"); // Remove a file to make sure that metadata is not reused from the -// previous commit +// previous commit. Remove that file from the expected metadata and +// replace the dirtree object. testDataDir.get_child('another-file').delete(null); delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; +delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51']; +expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138]; repo.prepare_transaction(null); mtree = OSTree.MutableTree.new(); From 291e9da258dc2c5b7cd5191f1c29e9b5f9d79b3b Mon Sep 17 00:00:00 2001 From: John Hiesey Date: Thu, 24 Oct 2019 09:10:57 -0600 Subject: [PATCH 16/49] lib/commit: Include object type in sizes metadata Append a byte encoding the OSTree object type for each object in the metadata. This allows the commit metadata to be fetched and then for the program to see which objects it already has for an accurate calculation of which objects need to be downloaded. This slightly breaks the `ostree.sizes` `ay` metadata entries. However, it's unlikely anyone was asserting the length of the entries since the array currently ends in 2 variable length integers. As far as I know, the only users of the sizes metadata are the ostree test suite and Endless' eos-updater[1]. The former is updated here and the latter already expects this format. 1. https://github.com/endlessm/eos-updater/ --- src/libostree/ostree-repo-commit.c | 10 +++-- tests/test-sizes.js | 64 +++++++++++++++++++----------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index f88e2d78..2294c846 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -322,16 +322,19 @@ commit_loose_regfile_object (OstreeRepo *self, /* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */ typedef struct { + OstreeObjectType objtype; goffset unpacked; goffset archived; } OstreeContentSizeCacheEntry; static OstreeContentSizeCacheEntry * -content_size_cache_entry_new (goffset unpacked, - goffset archived) +content_size_cache_entry_new (OstreeObjectType objtype, + goffset unpacked, + goffset archived) { OstreeContentSizeCacheEntry *entry = g_slice_new0 (OstreeContentSizeCacheEntry); + entry->objtype = objtype; entry->unpacked = unpacked; entry->archived = archived; @@ -399,7 +402,7 @@ repo_store_size_entry (OstreeRepo *self, repo_ensure_size_entries (self); g_hash_table_replace (self->object_sizes, g_strdup (checksum), - content_size_cache_entry_new (unpacked, archived)); + content_size_cache_entry_new (objtype, unpacked, archived)); } static int @@ -450,6 +453,7 @@ add_size_index_to_metadata (OstreeRepo *self, g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); _ostree_write_varuint64 (buffer, e_size->unpacked); + g_string_append_c (buffer, (gchar) e_size->objtype); g_variant_builder_add (&index_builder, "@ay", ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len)); diff --git a/tests/test-sizes.js b/tests/test-sizes.js index 685fe3fe..a2246536 100755 --- a/tests/test-sizes.js +++ b/tests/test-sizes.js @@ -67,15 +67,15 @@ function unpackByteArray(variant) { return array; } -function validateSizes(repo, commit, expectedFiles) { +function validateSizes(repo, commit, expectedObjects) { let [,commitVariant] = repo.load_variant(OSTree.ObjectType.COMMIT, commit); let metadata = commitVariant.get_child_value(0); let sizes = metadata.lookup_value('ostree.sizes', GLib.VariantType.new('aay')); - let nSizes = sizes.n_children(); - let expectedNSizes = Object.keys(expectedFiles).length - assertEquals(nSizes, expectedNSizes); + let nObjects = sizes.n_children(); + let expectedNObjects = Object.keys(expectedObjects).length + assertEquals(nObjects, expectedNObjects); - for (let i = 0; i < nSizes; i++) { + for (let i = 0; i < nObjects; i++) { let sizeEntry = sizes.get_child_value(i); assertGreaterEquals(sizeEntry.n_children(), 34); let entryBytes = unpackByteArray(sizeEntry); @@ -94,15 +94,20 @@ function validateSizes(repo, commit, expectedFiles) { assertGreaterEquals(remainingBytes.length, 1); [uncompressedSize, varintRead] = readVarint(remainingBytes); remainingBytes = remainingBytes.slice(varintRead); - assertEquals(remainingBytes.length, 0); + assertEquals(remainingBytes.length, 1); + let objectType = remainingBytes[0]; + let objectTypeString = OSTree.object_type_to_string(objectType); print("compressed = " + compressedSize); print("uncompressed = " + uncompressedSize); + print("objtype = " + objectTypeString + " (" + objectType + ")"); + let objectName = OSTree.object_to_string(checksumString, objectType); + print("object = " + objectName); - if (!(checksumString in expectedFiles)) { - throw new Error("Checksum " + checksumString + " not in " + - JSON.stringify(expectedFiles)); + if (!(objectName in expectedObjects)) { + throw new Error("Object " + objectName + " not in " + + JSON.stringify(expectedObjects)); } - let expectedSizes = expectedFiles[checksumString]; + let expectedSizes = expectedObjects[objectName]; let expectedCompressedSize = expectedSizes[0]; let expectedUncompressedSize = expectedSizes[1]; if (compressedSize != expectedCompressedSize) { @@ -152,15 +157,26 @@ print("commit => " + commit); repo.commit_transaction(null); -// Test the sizes metadata -let expectedFiles = { - 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad': [54, 18], - 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b': [48, 12], - '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0': [43, 9], - '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51': [185, 185], - '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488': [12, 12], +// Test the sizes metadata. The key is the object and the value is an +// array of compressed size and uncompressed size. +let expectedObjects = { + 'f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file': [ + 54, 18 + ], + 'd35bfc50864fca777dbeead3ba3689115b76674a093210316589b1fe5cc3ff4b.file': [ + 48, 12 + ], + '8322876a078e79d8c960b8b4658fe77e7b2f878f8a6cf89dbb87c6becc8128a0.file': [ + 43, 9 + ], + '1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree': [ + 185, 185 + ], + '446a0ef11b7cc167f3b603e585c7eeeeb675faa412d5ec73f62988eb0b6c5488.dirmeta': [ + 12, 12 + ], }; -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes"); @@ -168,9 +184,11 @@ print("ok test-sizes"); // previous commit. Remove that file from the expected metadata and // replace the dirtree object. testDataDir.get_child('another-file').delete(null); -delete expectedFiles['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad']; -delete expectedFiles['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51']; -expectedFiles['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539'] = [138, 138]; +delete expectedObjects['f5ee222a21e2c96edbd6f2543c4bc8a039f827be3823d04777c9ee187778f1ad.file']; +delete expectedObjects['1c77033ca06eae77ed99cb26472969964314ffd5b4e4c0fd3ff6ec4265c86e51.dirtree']; +expectedObjects['a384660cc18ffdb60296961dde9a2d6f78f4fec095165652cb53aa81f6dc7539.dirtree'] = [ + 138, 138 +]; repo.prepare_transaction(null); mtree = OSTree.MutableTree.new(); @@ -180,7 +198,7 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); print("commit => " + commit); repo.commit_transaction(null); -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes file deleted"); @@ -194,6 +212,6 @@ repo.write_directory_to_mtree(testDataDir, mtree, commitModifier, null); print("commit => " + commit); repo.commit_transaction(null); -validateSizes(repo, commit, expectedFiles); +validateSizes(repo, commit, expectedObjects); print("ok test-sizes repeated"); From 1bbe674d91a23e733e1bb58fc6c28c4ab86f94ae Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 24 Oct 2019 17:00:33 -0600 Subject: [PATCH 17/49] libarchive: Support commit sizes metadata Call the helper to set the generate_sizes boolean so that object size data is stored while writing the mtree. --- src/libostree/ostree-repo-commit.c | 10 +++++----- src/libostree/ostree-repo-libarchive.c | 2 ++ src/libostree/ostree-repo-private.h | 4 ++++ tests/test-libarchive.sh | 16 +++++++++++++++- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 2294c846..87b585fd 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -348,9 +348,9 @@ content_size_cache_entry_free (gpointer entry) g_slice_free (OstreeContentSizeCacheEntry, entry); } -static void -repo_setup_generate_sizes (OstreeRepo *self, - OstreeRepoCommitModifier *modifier) +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier) { if (modifier && modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES) { @@ -3923,7 +3923,7 @@ ostree_repo_write_directory_to_mtree (OstreeRepo *self, } else { - repo_setup_generate_sizes (self, modifier); + _ostree_repo_setup_generate_sizes (self, modifier); g_autoptr(GPtrArray) path = g_ptr_array_new (); if (!write_directory_to_mtree_internal (self, dir, mtree, modifier, path, @@ -3957,7 +3957,7 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error) { - repo_setup_generate_sizes (self, modifier); + _ostree_repo_setup_generate_sizes (self, modifier); g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error)) diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 1850f99f..d55459f4 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -844,6 +844,8 @@ ostree_repo_import_archive_to_mtree (OstreeRepo *self, .modifier = modifier }; + _ostree_repo_setup_generate_sizes (self, modifier); + while (TRUE) { int r = archive_read_next_header (a, &aictx.entry); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index bc2325e5..2864d81e 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -337,6 +337,10 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, GFileInfo *file_info, GFileInfo **out_modified_info); +void +_ostree_repo_setup_generate_sizes (OstreeRepo *self, + OstreeRepoCommitModifier *modifier); + gboolean _ostree_repo_remote_name_is_file (const char *remote_name); diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 24de55b2..174be800 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -28,7 +28,7 @@ fi . $(dirname $0)/libtest.sh -echo "1..17" +echo "1..18" setup_test_repository "bare" @@ -234,3 +234,17 @@ for filter in '^usr/bin/,usr/sbin/' '/bin/,/sbin/'; do assert_file_has_content usr/lib/libfoo.so 'a library' echo "ok tar pathname filter modification: ${filter}" done + +# Test sizes metadata. This needs an archive repo, so a separate repo is used. +cd ${test_tmpdir} +rm -rf repo2 +ostree_repo_init repo2 --mode=archive +${CMD_PREFIX} ostree --repo=repo2 commit \ + -s "from tar" -b test-tar \ + --generate-sizes \ + --tree=tar=foo.tar.gz +${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt +assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.1[  ]kB' +assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/900[  ]bytes' +assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12' +echo "ok tar sizes metadata" From fcbb453443c4f22e8621c8593877ab775e5a5884 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 19 Dec 2019 12:50:46 -0700 Subject: [PATCH 18/49] core: Add OstreeCommitSizesEntry type This will be used when reading out entries in the `ostree.sizes` metadata. Each entry corresponds to an object in the metadata array. --- apidoc/ostree-sections.txt | 6 +++ src/libostree/libostree-devel.sym | 5 +++ src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-core.c | 70 +++++++++++++++++++++++++++++ src/libostree/ostree-core.h | 32 +++++++++++++ 5 files changed, 114 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 1ef4bbf6..33035ca7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -151,7 +151,13 @@ ostree_validate_structureof_dirmeta ostree_commit_get_parent ostree_commit_get_timestamp ostree_commit_get_content_checksum +OstreeCommitSizesEntry +ostree_commit_sizes_entry_new +ostree_commit_sizes_entry_copy +ostree_commit_sizes_entry_free ostree_check_version + +ostree_commit_sizes_entry_get_type
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index d1666176..8e7473c5 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,11 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { +global: + ostree_commit_sizes_entry_copy; + ostree_commit_sizes_entry_free; + ostree_commit_sizes_entry_get_type; + ostree_commit_sizes_entry_new; ostree_sysroot_initialize; ostree_sysroot_is_booted; ostree_sysroot_set_mount_namespace_in_use; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index c07f88a8..c9692ebe 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -49,6 +49,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoDevInoCache, ostree_repo_devino_cache_u G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAsyncProgress, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBootconfigParser, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCommitSizesEntry, ostree_commit_sizes_entry_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeDeployment, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeGpgVerifyResult, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeKernelArgs, ostree_kernel_args_free) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 3d16757e..160f954f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -2430,6 +2430,76 @@ ostree_commit_get_content_checksum (GVariant *commit_variant) return g_strdup (hexdigest); } +G_DEFINE_BOXED_TYPE (OstreeCommitSizesEntry, ostree_commit_sizes_entry, + ostree_commit_sizes_entry_copy, ostree_commit_sizes_entry_free) + +/** + * ostree_commit_sizes_entry_new: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Create a new #OstreeCommitSizesEntry for representing an object in a + * commit's "ostree.sizes" metadata. + * + * Returns: (transfer full) (nullable): a new #OstreeCommitSizesEntry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived) +{ + g_return_val_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL), NULL); + + g_autoptr(OstreeCommitSizesEntry) entry = g_new0 (OstreeCommitSizesEntry, 1); + entry->checksum = g_strdup (checksum); + entry->objtype = objtype; + entry->unpacked = unpacked; + entry->archived = archived; + + return g_steal_pointer (&entry); +} + +/** + * ostree_commit_sizes_entry_copy: + * @entry: (not nullable): an #OstreeCommitSizesEntry + * + * Create a copy of the given @entry. + * + * Returns: (transfer full) (nullable): a new copy of @entry + * Since: 2019.7 + */ +OstreeCommitSizesEntry * +ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry) +{ + g_return_val_if_fail (entry != NULL, NULL); + + return ostree_commit_sizes_entry_new (entry->checksum, + entry->objtype, + entry->unpacked, + entry->archived); +} + +/** + * ostree_commit_sizes_entry_free: + * @entry: (transfer full): an #OstreeCommitSizesEntry + * + * Free given @entry. + * + * Since: 2019.7 + */ +void +ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) +{ + g_return_if_fail (entry != NULL); + + g_free (entry->checksum); + g_free (entry); +} + /* Used in pull/deploy to validate we're not being downgraded */ gboolean _ostree_compare_timestamps (const char *current_rev, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 69477a75..a61ae06c 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -521,6 +521,38 @@ guint64 ostree_commit_get_timestamp (GVariant *commit_variant); _OSTREE_PUBLIC gchar * ostree_commit_get_content_checksum (GVariant *commit_variant); +/** + * OstreeCommitSizesEntry: + * @checksum: (not nullable): object checksum + * @objtype: object type + * @unpacked: unpacked object size + * @archived: compressed object size + * + * Structure representing an entry in the "ostree.sizes" commit metadata. Each + * entry corresponds to an object in the associated commit. + * + * Since: 2019.5 + */ +typedef struct { + gchar *checksum; + OstreeObjectType objtype; + guint64 unpacked; + guint64 archived; +} OstreeCommitSizesEntry; + +_OSTREE_PUBLIC +GType ostree_commit_sizes_entry_get_type (void); + +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_new (const gchar *checksum, + OstreeObjectType objtype, + guint64 unpacked, + guint64 archived); +_OSTREE_PUBLIC +OstreeCommitSizesEntry *ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry); +_OSTREE_PUBLIC +void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry); + _OSTREE_PUBLIC gboolean ostree_check_version (guint required_year, guint required_release); From 260bcd11938be8fbe49846eff913206e5df4168a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 20 Jan 2020 19:54:00 -0700 Subject: [PATCH 19/49] core: Add ostree_commit_get_object_sizes API This function parses the object listing in the `ostree.sizes` metadata and returns an array of `OstreeCommitSizesEntry` structures. Unfortunately, for reasons I don't understand, the linker wants to resolve `_ostree_read_varuint64` from `ostree-core.c` even though it's not used by `test-checksum.c` at all. --- Makefile-tests.am | 5 +- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-core-private.h | 3 + src/libostree/ostree-core.c | 112 ++++++++++++++++++++++++++++ src/libostree/ostree-core.h | 5 ++ src/libostree/ostree-repo-private.h | 2 - 7 files changed, 126 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index fc2f2d91..a4acb8e0 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -347,7 +347,10 @@ tests_test_varint_LDADD = $(TESTS_LDADD) tests_test_bsdiff_CFLAGS = $(TESTS_CFLAGS) tests_test_bsdiff_LDADD = libbsdiff.la $(TESTS_LDADD) -tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c +tests_test_checksum_SOURCES = \ + src/libostree/ostree-core.c \ + src/libostree/ostree-varint.c \ + tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 33035ca7..32cf5228 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -151,6 +151,7 @@ ostree_validate_structureof_dirmeta ostree_commit_get_parent ostree_commit_get_timestamp ostree_commit_get_content_checksum +ostree_commit_get_object_sizes OstreeCommitSizesEntry ostree_commit_sizes_entry_new ostree_commit_sizes_entry_copy diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 8e7473c5..ff5f52c4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2019.7 { global: + ostree_commit_get_object_sizes; ostree_commit_sizes_entry_copy; ostree_commit_sizes_entry_free; ostree_commit_sizes_entry_get_type; diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 43cf22c4..c1a82386 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -102,6 +102,9 @@ _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) */ #define _OSTREE_LOOSE_PATH_MAX (256) +/* GVariant format for ostree.sizes metadata entries. */ +#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" + char * _ostree_get_relative_object_path (const char *checksum, OstreeObjectType type, diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 160f954f..4667dd8f 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -32,6 +32,7 @@ #include "ostree.h" #include "ostree-core-private.h" #include "ostree-chain-input-stream.h" +#include "ostree-varint.h" #include "otutil.h" /* Generic ABI checks */ @@ -2500,6 +2501,117 @@ ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) g_free (entry); } +static gboolean +read_sizes_entry (GVariant *entry, + OstreeCommitSizesEntry **out_sizes, + GError **error) +{ + gsize entry_size = g_variant_get_size (entry); + g_return_val_if_fail (entry_size >= OSTREE_SHA256_DIGEST_LEN + 2, FALSE); + + const guchar *buffer = g_variant_get_data (entry); + if (buffer == NULL) + return glnx_throw (error, "Could not read ostree.sizes metadata entry"); + + char checksum[OSTREE_SHA256_STRING_LEN + 1]; + ostree_checksum_inplace_from_bytes (buffer, checksum); + buffer += OSTREE_SHA256_DIGEST_LEN; + entry_size -= OSTREE_SHA256_DIGEST_LEN; + + gsize bytes_read = 0; + guint64 archived = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &archived, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + guint64 unpacked = 0; + if (!_ostree_read_varuint64 (buffer, entry_size, &unpacked, &bytes_read)) + return glnx_throw (error, "Unexpected EOF reading ostree.sizes varint"); + buffer += bytes_read; + entry_size -= bytes_read; + + /* On newer commits, an additional byte is used for the object type. */ + OstreeObjectType objtype; + if (entry_size > 0) + { + objtype = *buffer; + if (objtype < OSTREE_OBJECT_TYPE_FILE || objtype > OSTREE_OBJECT_TYPE_LAST) + return glnx_throw (error, "Unexpected ostree.sizes object type %u", + objtype); + buffer++; + entry_size--; + } + else + { + /* Assume the object is a file. */ + objtype = OSTREE_OBJECT_TYPE_FILE; + } + + g_autoptr(OstreeCommitSizesEntry) sizes = ostree_commit_sizes_entry_new (checksum, + objtype, + unpacked, + archived); + + if (out_sizes != NULL) + *out_sizes = g_steal_pointer (&sizes); + + return TRUE; +} + +/** + * ostree_commit_get_object_sizes: + * @commit_variant: (not nullable): variant of type %OSTREE_OBJECT_TYPE_COMMIT + * @out_sizes_entries: (out) (element-type OstreeCommitSizesEntry) (transfer container) (optional): + * return location for an array of object size entries + * @error: Error + * + * Reads a commit's "ostree.sizes" metadata and returns an array of + * #OstreeCommitSizesEntry in @out_sizes_entries. Each element + * represents an object in the commit. If the commit does not contain + * the "ostree.sizes" metadata, a %G_IO_ERROR_NOT_FOUND error will be + * returned. + * + * Since: 2019.7 + */ +gboolean +ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error) +{ + g_return_val_if_fail (commit_variant != NULL, FALSE); + + g_autoptr(GVariant) metadata = g_variant_get_child_value (commit_variant, 0); + g_autoptr(GVariant) sizes_variant = + g_variant_lookup_value (metadata, "ostree.sizes", + G_VARIANT_TYPE ("a" _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE)); + if (sizes_variant == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No metadata key ostree.sizes in commit"); + return FALSE; + } + + g_autoptr(GPtrArray) sizes_entries = + g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_commit_sizes_entry_free); + g_autoptr(GVariant) entry = NULL; + GVariantIter entry_iter; + g_variant_iter_init (&entry_iter, sizes_variant); + while ((entry = g_variant_iter_next_value (&entry_iter))) + { + OstreeCommitSizesEntry *sizes_entry = NULL; + if (!read_sizes_entry (entry, &sizes_entry, error)) + return FALSE; + g_clear_pointer (&entry, g_variant_unref); + g_ptr_array_add (sizes_entries, sizes_entry); + } + + if (out_sizes_entries != NULL) + *out_sizes_entries = g_steal_pointer (&sizes_entries); + + return TRUE; +} + /* Used in pull/deploy to validate we're not being downgraded */ gboolean _ostree_compare_timestamps (const char *current_rev, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index a61ae06c..10601123 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -553,6 +553,11 @@ OstreeCommitSizesEntry *ostree_commit_sizes_entry_copy (const OstreeCommitSizesE _OSTREE_PUBLIC void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry); +_OSTREE_PUBLIC +gboolean ostree_commit_get_object_sizes (GVariant *commit_variant, + GPtrArray **out_sizes_entries, + GError **error); + _OSTREE_PUBLIC gboolean ostree_check_version (guint required_year, guint required_release); diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 2864d81e..0465327c 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -31,8 +31,6 @@ G_BEGIN_DECLS #define OSTREE_DELTAPART_VERSION (0) -#define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" - #define _OSTREE_SUMMARY_CACHE_DIR "summaries" #define _OSTREE_CACHE_DIR "cache" From 97c831dd5fe509483939cdd40703a0ce8e0e9bfd Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 24 Oct 2019 15:21:49 -0600 Subject: [PATCH 20/49] bin/show: Add --print-sizes option to show sizes metadata Use the new `ostree_commit_get_object_sizes()` API to read the `ostree.sizes` commit metadata and print a summary. --- Makefile-tests.am | 1 + bash/ostree | 1 + man/ostree-show.xml | 11 ++++++ src/ostree/ot-builtin-show.c | 69 ++++++++++++++++++++++++++++++++++++ tests/test-pull-sizes.sh | 58 ++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100755 tests/test-pull-sizes.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index a4acb8e0..615b8480 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -86,6 +86,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-resume.sh \ tests/test-pull-basicauth.sh \ tests/test-pull-repeated.sh \ + tests/test-pull-sizes.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ tests/test-pull-localcache.sh \ diff --git a/bash/ostree b/bash/ostree index fc429983..4aec588b 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1445,6 +1445,7 @@ _ostree_show() { local boolean_options=" $main_boolean_options --print-related + --print-sizes --raw " diff --git a/man/ostree-show.xml b/man/ostree-show.xml index a3d9aa4a..a28f704c 100644 --- a/man/ostree-show.xml +++ b/man/ostree-show.xml @@ -99,6 +99,17 @@ Boston, MA 02111-1307, USA. + + + + + Show the commit size metadata. This in only supported for + commits that contain ostree.sizes + metadata. This can be included when creating commits with + ostree commit --generate-sizes. + + + diff --git a/src/ostree/ot-builtin-show.c b/src/ostree/ot-builtin-show.c index 5091a93c..96e2d4c6 100644 --- a/src/ostree/ot-builtin-show.c +++ b/src/ostree/ot-builtin-show.c @@ -33,6 +33,7 @@ static gboolean opt_print_related; static char* opt_print_variant_type; static char* opt_print_metadata_key; static char* opt_print_detached_metadata_key; +static gboolean opt_print_sizes; static gboolean opt_raw; static gboolean opt_no_byteswap; static char *opt_gpg_homedir; @@ -48,6 +49,7 @@ static GOptionEntry options[] = { { "print-variant-type", 0, 0, G_OPTION_ARG_STRING, &opt_print_variant_type, "Memory map OBJECT (in this case a filename) to the GVariant type string", "TYPE" }, { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key", "KEY" }, { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key", "KEY" }, + { "print-sizes", 0, 0, G_OPTION_ARG_NONE, &opt_print_sizes, "Show the commit size metadata", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" }, { "no-byteswap", 'B', 0, G_OPTION_ARG_NONE, &opt_no_byteswap, "Do not automatically convert variant data from big endian" }, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, @@ -146,6 +148,65 @@ do_print_metadata_key (OstreeRepo *repo, return TRUE; } +static gboolean +do_print_sizes (OstreeRepo *repo, + const char *rev, + GError **error) +{ + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, + &commit, error)) + { + g_prefix_error (error, "Failed to read commit: "); + return FALSE; + } + + g_autoptr(GPtrArray) sizes = NULL; + if (!ostree_commit_get_object_sizes (commit, &sizes, error)) + return FALSE; + + gint64 new_archived = 0; + gint64 new_unpacked = 0; + gsize new_objects = 0; + gint64 archived = 0; + gint64 unpacked = 0; + gsize objects = 0; + for (guint i = 0; i < sizes->len; i++) + { + OstreeCommitSizesEntry *entry = sizes->pdata[i]; + + archived += entry->archived; + unpacked += entry->unpacked; + objects++; + + gboolean exists; + if (!ostree_repo_has_object (repo, entry->objtype, entry->checksum, + &exists, NULL, error)) + return FALSE; + + if (!exists) + { + /* Object not in local repo */ + new_archived += entry->archived; + new_unpacked += entry->unpacked; + new_objects++; + } + } + + g_autofree char *new_archived_str = g_format_size (new_archived); + g_autofree char *archived_str = g_format_size (archived); + g_autofree char *new_unpacked_str = g_format_size (new_unpacked); + g_autofree char *unpacked_str = g_format_size (unpacked); + g_print ("Compressed size (needed/total): %s/%s\n" + "Unpacked size (needed/total): %s/%s\n" + "Number of objects (needed/total): %" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "\n", + new_archived_str, archived_str, + new_unpacked_str, unpacked_str, + new_objects, objects); + + return TRUE; +} + static gboolean print_object (OstreeRepo *repo, OstreeObjectType objtype, @@ -279,6 +340,14 @@ ostree_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, if (!do_print_variant_generic (G_VARIANT_TYPE (opt_print_variant_type), rev, error)) return FALSE; } + else if (opt_print_sizes) + { + if (!ostree_repo_resolve_rev (repo, rev, FALSE, &resolved_rev, error)) + return FALSE; + + if (!do_print_sizes (repo, resolved_rev, error)) + return FALSE; + } else { gboolean found = FALSE; diff --git a/tests/test-pull-sizes.sh b/tests/test-pull-sizes.sh new file mode 100755 index 00000000..8ee07cc8 --- /dev/null +++ b/tests/test-pull-sizes.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (C) 2019 Endless Mobile, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" "--generate-sizes" + +echo '1..3' + +cd ${test_tmpdir} +mkdir repo +ostree_repo_init repo +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + +# Pull commit metadata only. All size and objects will be needed. +${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 637[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 457[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 10/10' +echo "ok sizes commit metadata only" + +# Pull the parent commit so we get most of the objects +parent=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main^) +${CMD_PREFIX} ostree --repo=repo pull origin ${parent} +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 501[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 429[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 6/10' +echo "ok sizes commit partial" + +# Finish pulling the commit and check that no objects needed +${CMD_PREFIX} ostree --repo=repo pull origin main +${CMD_PREFIX} ostree --repo=repo show --print-sizes origin:main > show.txt +assert_file_has_content show.txt 'Compressed size (needed/total): 0[  ]bytes/637[  ]bytes' +assert_file_has_content show.txt 'Unpacked size (needed/total): 0[  ]bytes/457[  ]bytes' +assert_file_has_content show.txt 'Number of objects (needed/total): 0/10' +echo "ok sizes commit full" From 5135a1e58ade2bfafc8c1fda359540eafd72531e Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 21 Jan 2020 10:25:17 -0700 Subject: [PATCH 21/49] tests/core: Really pick C.UTF-8 locale The case-ignoring regex `^(C|en_US)` will match any locale that starts with `c`. On my system this is `ca_AD.utf8`, which breaks the test suite. Instead, use a single regex that includes the joining `.` rather than 2 separate regexes. This also changes `head` to use the `-n` option, which has been preferred for at least 10 years in the coreutils version and is supported by busybox as well. --- tests/libtest-core.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index 46aafab0..945d2857 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -41,7 +41,7 @@ assert_not_reached () { # If we can't find the locale command assume we have support for C.UTF-8 # (e.g. musl based systems) if type -p locale >/dev/null; then - export LC_ALL=$(locale -a | grep -Ee '\.(UTF-8|utf8)' | grep -iEe '^(C|en_US)' | head -1 || true) + export LC_ALL=$(locale -a | grep -iEe '^(C|en_US)\.(UTF-8|utf8)$' | head -n1 || true) if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi else export LC_ALL=C.UTF-8 From a6994459c198e225693a5349d23dba3825ccda9c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 21 Jan 2020 14:56:31 -0700 Subject: [PATCH 22/49] ci/rpmostree: Bump to 2019.4 The vmcheck tests in 2019.3 fail because of an SSH control socket issue on overlayfs. This is fixed in 2019.4[1]. That has some other changes such as using Python 3 in tests. The package dependencies have been synced from the rpm-ostree CI for that. Unfortunately, this is no longer a totally representative test of f29 since it has 2019.3 in updates. But that's the price you pay for exercising someone else's CI from your own CI. 1. https://github.com/coreos/rpm-ostree/commit/c89f81c1385ef095616b0e7001926572a20057b2 Fixes: #1994 --- ci/rpmostree.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ci/rpmostree.sh b/ci/rpmostree.sh index 76841ffe..27fefaf6 100755 --- a/ci/rpmostree.sh +++ b/ci/rpmostree.sh @@ -6,7 +6,7 @@ set -xeuo pipefail # Frozen to a tag for now to help predictability; it's # also useful to test building *older* versions since # that must work. -RPMOSTREE_TAG=v2019.3 +RPMOSTREE_TAG=v2019.4 dn=$(dirname $0) . ${dn}/libpaprci/libbuild.sh @@ -18,12 +18,14 @@ pkg_install_buildroot pkg_builddep ostree rpm-ostree pkg_install rpm-ostree && rpm -e rpm-ostree -# Duplicate of deps from build.sh in rpm-ostree for tests -pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq PyYAML \ - libubsan libasan libtsan elfutils fuse sudo python-gobject-base \ - selinux-policy-devel selinux-policy-targeted openssh-clients ansible -# This one is in the papr.yml -pkg_install rsync +# Duplicate of deps from ci/installdeps.sh in rpm-ostree for tests +pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq python3-pyyaml \ + libubsan libasan libtsan elfutils fuse sudo python3-gobject-base \ + selinux-policy-devel selinux-policy-targeted python3-createrepo_c \ + rsync python3-rpm parallel clang rustfmt-preview + +# From rpm-ostree/ci/vmcheck-provision.sh +pkg_install openssh-clients ansible # build+install ostree cd ${codedir} From c54a3b5daa82c2bb791f061a54cdfd685f3f22be Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 19 Jun 2019 13:08:40 -0500 Subject: [PATCH 23/49] lib/gpg: Prefer declare-and-initialize style As noted in https://github.com/ostreedev/ostree/pull/1872#discussion_r295408768. --- src/libostree/ostree-gpg-verify-result.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index d4d1cef6..63c3394a 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -548,14 +548,10 @@ append_expire_info (GString *output_buffer, gint64 exp_timestamp, gboolean expired) { - g_autoptr(GDateTime) date_time_utc = NULL; - g_autoptr(GDateTime) date_time_local = NULL; - g_autofree char *formatted_date_time = NULL; - if (line_prefix != NULL) g_string_append (output_buffer, line_prefix); - date_time_utc = g_date_time_new_from_unix_utc (exp_timestamp); + g_autoptr(GDateTime) date_time_utc = g_date_time_new_from_unix_utc (exp_timestamp); if (date_time_utc == NULL) { g_string_append_printf (output_buffer, @@ -565,8 +561,8 @@ append_expire_info (GString *output_buffer, return; } - date_time_local = g_date_time_to_local (date_time_utc); - formatted_date_time = g_date_time_format (date_time_local, "%c"); + g_autoptr(GDateTime) date_time_local = g_date_time_to_local (date_time_utc); + g_autofree char *formatted_date_time = g_date_time_format (date_time_local, "%c"); if (expired) { From 7f04c5d76412e99908e92111902d0d8dc0c8300d Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 19 Jun 2019 09:49:32 -0500 Subject: [PATCH 24/49] tests/libtest: Record long GPG key IDs and fingerprints Use long GPG key IDs as it's safer and matches the format used by gpg and gpgme. Add the associated fingerprints since these are needed by gpg when manipulating keys. --- tests/libtest.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index cbdf331c..00e69b12 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -78,9 +78,12 @@ unset TAR_OPTIONS # easily clean up. export OSTREE_SYSROOT_DEBUG=mutable-deployments -export TEST_GPG_KEYID_1="472CDAFA" -export TEST_GPG_KEYID_2="CA950D41" -export TEST_GPG_KEYID_3="DF444D67" +export TEST_GPG_KEYID_1="7FCA23D8472CDAFA" +export TEST_GPG_KEYFPR_1="5E65DE75AB1C501862D476347FCA23D8472CDAFA" +export TEST_GPG_KEYID_2="D8228CFECA950D41" +export TEST_GPG_KEYFPR_2="7B3B1020D74479687FDB2273D8228CFECA950D41" +export TEST_GPG_KEYID_3="0D15FAE7DF444D67" +export TEST_GPG_KEYFPR_3="7D29CF060B8269CDF63BFBDD0D15FAE7DF444D67" # GPG when creating signatures demands a writable # homedir in order to create lockfiles. Work around From 63414e85c350270517eba4c2cb22b31629809d35 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 19 Jun 2019 11:51:47 -0500 Subject: [PATCH 25/49] tests/libtest: Make temporary gpghome private gpg prints a warning about unsafe permissions if the homedir is group or world readable. This is just noise in the test logs, so appease it by making the homedir 700. --- tests/libtest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 00e69b12..c82bf487 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -85,12 +85,13 @@ export TEST_GPG_KEYFPR_2="7B3B1020D74479687FDB2273D8228CFECA950D41" export TEST_GPG_KEYID_3="0D15FAE7DF444D67" export TEST_GPG_KEYFPR_3="7D29CF060B8269CDF63BFBDD0D15FAE7DF444D67" -# GPG when creating signatures demands a writable +# GPG when creating signatures demands a private writable # homedir in order to create lockfiles. Work around # this by copying locally. echo "Copying gpghome to ${test_tmpdir}" cp -a "${test_srcdir}/gpghome" ${test_tmpdir} chmod -R u+w "${test_tmpdir}" +chmod 700 "${test_tmpdir}/gpghome" export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted From b825083549994dd577c45897cff50052db5506db Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 21 Jun 2019 06:27:33 -0500 Subject: [PATCH 26/49] tests/gpghome: Create revocation certificates for keys These can then be imported during a test to revoke a key without trying to go through the gpg --generate-revocation dialog. Note that these need to go in a subdirectory of the homedir since `gpgkeypath` will try to import every regular file in the homedir. --- Makefile-tests.am | 6 ++++++ tests/gpghome/revocations/key1.rev | 12 ++++++++++++ tests/gpghome/revocations/key2.rev | 12 ++++++++++++ tests/gpghome/revocations/key3.rev | 12 ++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/gpghome/revocations/key1.rev create mode 100644 tests/gpghome/revocations/key2.rev create mode 100644 tests/gpghome/revocations/key3.rev diff --git a/Makefile-tests.am b/Makefile-tests.am index fc2f2d91..ef71df5c 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -220,6 +220,12 @@ dist_gpginsttest_DATA = tests/gpghome/secring.gpg \ tests/gpghome/key3.asc gpginsttest_trusteddir = $(installed_testdir)/gpghome/trusted dist_gpginsttest_trusted_DATA = tests/gpghome/trusted/pubring.gpg +gpginsttest_revocdir = $(installed_testdir)/gpghome/revocations +dist_gpginsttest_revoc_DATA = \ + tests/gpghome/revocations/key1.rev \ + tests/gpghome/revocations/key2.rev \ + tests/gpghome/revocations/key3.rev \ + $(NULL) gpgvinsttestdir = $(installed_testdir)/gpg-verify-data dist_gpgvinsttest_DATA = $(addprefix tests/gpg-verify-data/, \ diff --git a/tests/gpghome/revocations/key1.rev b/tests/gpghome/revocations/key1.rev new file mode 100644 index 00000000..b918151e --- /dev/null +++ b/tests/gpghome/revocations/key1.rev @@ -0,0 +1,12 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: This is a revocation certificate + +iQE9BCABCgAnFiEEXmXedascUBhi1HY0f8oj2Ecs2voFAl0MvqgJHQBUZXN0aW5n +AAoJEH/KI9hHLNr6/dQH/iPZjfJgFAc/TIR4xE4kB0yL4zdMqxgV1ef/atQDLEN4 +MBiqIltzb8WyG+cpNfNZgFmqXmCRN+1IAla9piixe76ZwOqcQ6S5MU/8nMcyMsD9 +edg+9sg0DH8SEzejVma3ZLfaJ/6ZpU7c6a4vCPNcRBC7PxAvAc0LnAN6KQYGU7GR +gv2k/JsGYgvmUAajhVFy0jc4jGkhRBHMDksGGFdYY94RATFF4gWtlUXyRYMTXBCf +eM3bxEeSMbU7lXCZg9zjoxP6XzJuNW1SLz3zL90GnO19uNh8Pf6pHmkCNTO/L1Ua +Cc6fb3ubtdqgs6an84U/aod1VcK7BNASqZ2gYUsF2KE= +=owvo +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/gpghome/revocations/key2.rev b/tests/gpghome/revocations/key2.rev new file mode 100644 index 00000000..9b8960a2 --- /dev/null +++ b/tests/gpghome/revocations/key2.rev @@ -0,0 +1,12 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: This is a revocation certificate + +iQE9BCABCgAnFiEEezsQINdEeWh/2yJz2CKM/sqVDUEFAl0Mvs4JHQBUZXN0aW5n +AAoJENgijP7KlQ1BzH4IAMUoTrW6XraDYq/d/b0qa3sZ1NTBPUXLp4gFaedZwKk/ +AKSUCK05RWRQO3HrSvmhdSUF6/9tFLGpbu7P56ihjAnq2vpzRyeNTEGQ02IzfCpM +SJup0R6iA7KmjiDutDoEgjhAzxCKbnU71SQ3PmjyaQT1KCBCDJVptcY6HDbo0dRN +vcjTfjtFqkPgqbHyXwGv3rlm4uctSVfACrOS/fKF4Q9Fs4prsUXjQpGLaTHdxhiL +pMRCTfZ4DBEMwAY7s9FpMpIh9rdOwE+zkv5CsE0uJVZq0WW5r0CBDCta6Sopt6uk +pIA+QHL9GPOrG2E3SJxyIBC37Sl40MGAJQ1djmecIGk= +=0KEe +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/gpghome/revocations/key3.rev b/tests/gpghome/revocations/key3.rev new file mode 100644 index 00000000..66e238a1 --- /dev/null +++ b/tests/gpghome/revocations/key3.rev @@ -0,0 +1,12 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: This is a revocation certificate + +iQE9BCABCgAnFiEEfSnPBguCac32O/vdDRX6599ETWcFAl0MvuYJHQBUZXN0aW5n +AAoJEA0V+uffRE1n/p8H/3mmSK2gtbxJ5sfu1z44ra82fLRAUupJzf53dAvvJCEK +4RSJFtHYu+hoPVFd9bmToxo2YQWe67MMZW7cHtq9D/a755SYOrty//KpXsGS22W/ +ZGatBjl36zuE6BoR18Q6VAMgVBwovPSlSuCEW+Ro8JZYyA/LbA95AnMprNod6Jw9 +VSsGC39au5rUlhEOHLL1Iw3dl4blxa6tf/roljbXzaN+Qh2/ez7Cy532oocak2FL +bbblBGrIdfYLAXpLqhnQk2vgEHZ+ZylvStBndpLWwEskXhmaHpW7+WapFhLCUOr+ +arzbc9XQ7ghhF9hSoKiToJqU5PRjaOex85BEDwE5gWY= +=ykAF +-----END PGP PUBLIC KEY BLOCK----- From 7fe265b08706677ae99512402dfaaa3830087bdf Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 19 Jun 2019 16:31:18 -0500 Subject: [PATCH 27/49] tests/gpg-verify-data: Split out signature data The full block with all 5 signatures remains, but this allows passing individual signatures through the GPG verification APIs. The split was done with `gpgsplit`, and looking at the output of `gpg --list-packets` of the split and unsplit files appears correct. --- Makefile-tests.am | 3 ++- tests/gpg-verify-data/README.md | 6 ++++-- tests/gpg-verify-data/lgpl2.sig0 | Bin 0 -> 287 bytes tests/gpg-verify-data/lgpl2.sig1 | Bin 0 -> 287 bytes tests/gpg-verify-data/lgpl2.sig2 | Bin 0 -> 287 bytes tests/gpg-verify-data/lgpl2.sig3 | Bin 0 -> 287 bytes tests/gpg-verify-data/lgpl2.sig4 | Bin 0 -> 293 bytes 7 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 tests/gpg-verify-data/lgpl2.sig0 create mode 100644 tests/gpg-verify-data/lgpl2.sig1 create mode 100644 tests/gpg-verify-data/lgpl2.sig2 create mode 100644 tests/gpg-verify-data/lgpl2.sig3 create mode 100644 tests/gpg-verify-data/lgpl2.sig4 diff --git a/Makefile-tests.am b/Makefile-tests.am index ef71df5c..553f535c 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -229,7 +229,8 @@ dist_gpginsttest_revoc_DATA = \ gpgvinsttestdir = $(installed_testdir)/gpg-verify-data dist_gpgvinsttest_DATA = $(addprefix tests/gpg-verify-data/, \ - gpg.conf lgpl2 lgpl2.sig pubring.gpg secring.gpg trustdb.gpg) + gpg.conf lgpl2 lgpl2.sig lgpl2.sig0 lgpl2.sig1 lgpl2.sig2 lgpl2.sig3 \ + lgpl2.sig4 pubring.gpg secring.gpg trustdb.gpg) endif endif diff --git a/tests/gpg-verify-data/README.md b/tests/gpg-verify-data/README.md index d96fbad5..9ca47581 100644 --- a/tests/gpg-verify-data/README.md +++ b/tests/gpg-verify-data/README.md @@ -1,5 +1,7 @@ This is a GPG config directory for use with the OstreeGpgVerifyResult -test cases. The test data (`lgplv2`) is signed with a variety of valid -and invalid GPG keys in a detached signature file (`lgplv2.sig`). +test cases. The test data (`lgpl2`) is signed with a variety of valid +and invalid GPG keys in a detached signature file (`lgpl2.sig`). In +addition, each detached signature is available in a separate file +(`lgpgl2.sig`). The passphrase for all the keys is `redhat`. diff --git a/tests/gpg-verify-data/lgpl2.sig0 b/tests/gpg-verify-data/lgpl2.sig0 new file mode 100644 index 0000000000000000000000000000000000000000..375c650ad95a3d4f442022fbf9a6f73b141f3b24 GIT binary patch literal 287 zcmV+)0pR|L0UQJX0RjL91p-wH_JjZm2@vSBTWp$9unXp)2mqLC101HD8!%VyL5jOP z7o6Ol1S0@ZCuue|tUleVI6!3PmM|Vhkk( z2M4B5{Wf`S3c%y<}(3~$&TL2O9-G1}}Je}NTfWDVZnM3SX(ic=4 zRmP~4k;n@U#G<@ktg$}fkp8Y$yEfgfXM&= literal 0 HcmV?d00001 diff --git a/tests/gpg-verify-data/lgpl2.sig1 b/tests/gpg-verify-data/lgpl2.sig1 new file mode 100644 index 0000000000000000000000000000000000000000..83a6227c9b4d771c4a3bfd81ef2d6551248e79cd GIT binary patch literal 287 zcmV+)0pR|L0UQJX0RjL91p-wH_vQc!2@om5Lofxiwq~i42mUwA9~;H4etgyK*BbRX zs>o7Tc>}bVB#32Ty;%|=bC8*y{xtH;V^TcS#KITJPI~DRSuCC;1^Bds>%-aT4DltN zP@G0%x%kOVyKnrS&Xf=B$D(zAn(-SH@>-_LZg)0NmFEXmaA7q*Ax${kwj&ISt&y7fciLN0-L<(njVjn%{_Uy6Y-_DhqaWwch|FA^k~CQg4FE lw>>3U=nhMguM=YT` zRQlDwyCGT1#t~}MtzF2jL8q zki1b7fTkjLx+gcai~C>$~yU2Zll^9)RUwo|L{_y8-huAEifwbDs|s zG7K^iU$gYn!trH~4M4&w9`V2Pm1(ML_Rz@-Y?%t8I02KfN{LvhncV`W+MsXSD%C2L zo2Cs9@ Date: Fri, 21 Jun 2019 07:09:43 -0500 Subject: [PATCH 28/49] tests/gpg-verify-data: Empty out trustdb.gpg When the private keys were generated, gpg added an ultimate trust entry since you normally want to trust your own keys. However, this throws off the expired signature testing since gpgme considers it valid if the key is fully or ultimately trusted. The use of a trustdb for the test-gpg-verify-result is unlike any other GPG verification in ostree. Under normal circumstances, a temporary GPG homedir is created without any trust information, so all keys are treated as having unknown trust. Regenerate an empty trustdb.gpg in gpg-verify-data so that the tests behave as ostree normally operates. After this the expired signature testing correctly shows up as a non-valid signature. The trustdb was regenerated by simply removing it and running any gpg operation with the gpg-verify-data directory as the homedir. --- tests/gpg-verify-data/trustdb.gpg | Bin 1520 -> 1200 bytes tests/test-gpg-verify-result.c | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gpg-verify-data/trustdb.gpg b/tests/gpg-verify-data/trustdb.gpg index 3f046fad106df32b0a4d73339c21fbeeb034656f..91f87170b0afbdbca5570ab736cce423a2e11e0e 100644 GIT binary patch delta 40 wcmeysy@6AJF})z2nVFH5k%@sJmgj`+L__h3>XRlq@NfK($vjbked31-0M}RyLI3~& literal 1520 zcmZQfFGy!*W@Ke#Vqgg6`Yq0Y9WZiX7sn8#n>r;_t1;|V0LrVNssZsZ)L|+Ci;Swn z9x$WfG8!(R5FnTiyrIXc@0^yX+M*Sq2~~$s#=yg%{%)O|`aAv)SHe7tG!Ho&ux{R# z234nsP{_;hE@7MZ_w&&KN4NM29N;@OOWQ>D5;pS`9`gK3JT+(E?cdC61>-Lioj95> z8ETF)vUz7dvYre((XVjr*1gi>zt7q9@gG$~$irAX3>vzj5wC=Dw$GHvob^02b9P@{ m(Hy9(Doho~zvV?43h!=*I7>xFSIzR0oBDcRbsj<<#sUCc*;Up6 diff --git a/tests/test-gpg-verify-result.c b/tests/test-gpg-verify-result.c index 95de1873..8b409ab5 100644 --- a/tests/test-gpg-verify-result.c +++ b/tests/test-gpg-verify-result.c @@ -115,7 +115,7 @@ test_check_counts (TestFixture *fixture, count_valid = ostree_gpg_verify_result_count_valid (fixture->result); g_assert_cmpint (count_all, ==, 5); - g_assert_cmpint (count_valid, ==, 2); + g_assert_cmpint (count_valid, ==, 1); } static void @@ -373,7 +373,7 @@ test_expired_signature (TestFixture *fixture, &key_missing, &key_exp_timestamp); - g_assert_true (valid); + g_assert_false (valid); g_assert_true (sig_expired); g_assert_false (key_expired); g_assert_false (key_revoked); From 2c24f28ce45fe9c416650f004e30bcb97c76e0f0 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 20 Jun 2019 09:32:13 -0500 Subject: [PATCH 29/49] tests/test-gpg-verify-result: Allow specifying signature files Currently tests are always run against the full lgpl2.sig file with all signatures, but it should also be possible to specify one or more of the individual lgpgl2.sig files. Drop the current usage of passing the signature index in the test data since it's always specific to the test function and instead provide an optional array of signature files for the test fixture to sign with. --- tests/test-gpg-verify-result.c | 72 ++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/tests/test-gpg-verify-result.c b/tests/test-gpg-verify-result.c index 8b409ab5..1b4e54c1 100644 --- a/tests/test-gpg-verify-result.c +++ b/tests/test-gpg-verify-result.c @@ -53,12 +53,12 @@ static void test_fixture_setup (TestFixture *fixture, gconstpointer user_data) { + const char * const *sig_files = user_data; gpgme_error_t gpg_error; gpgme_data_t data_buffer; gpgme_data_t signature_buffer; OstreeGpgVerifyResult *result; g_autofree char *homedir = NULL; - g_autofree char *filename = NULL; GError *local_error = NULL; /* Mimic what OstreeGpgVerifier does to create OstreeGpgVerifyResult. @@ -74,15 +74,47 @@ test_fixture_setup (TestFixture *fixture, NULL, &local_error, NULL); g_assert_no_error (local_error); - filename = g_build_filename (homedir, "lgpl2", NULL); - gpg_error = gpgme_data_new_from_file (&data_buffer, filename, 1); - assert_no_gpg_error (gpg_error, filename); + g_autofree char *data_filename = g_build_filename (homedir, "lgpl2", NULL); + gpg_error = gpgme_data_new_from_file (&data_buffer, data_filename, 1); + assert_no_gpg_error (gpg_error, data_filename); - g_clear_pointer (&filename, g_free); + if (sig_files == NULL) + { + /* No signature files specified, use full lgpl2.sig file */ + g_autofree char *filename = g_build_filename (homedir, "lgpl2.sig", NULL); + gpg_error = gpgme_data_new_from_file (&signature_buffer, filename, 1); + assert_no_gpg_error (gpg_error, filename); + } + else + { + /* Read all the specified files into the signature buffer */ + gpg_error = gpgme_data_new (&signature_buffer); + assert_no_gpg_error (gpg_error, NULL); - filename = g_build_filename (homedir, "lgpl2.sig", NULL); - gpg_error = gpgme_data_new_from_file (&signature_buffer, filename, 1); - assert_no_gpg_error (gpg_error, filename); + for (const char * const *name = sig_files; *name != NULL; name++) + { + g_autofree char *path = g_build_filename (homedir, *name, NULL); + g_autoptr(GFile) sig_file = g_file_new_for_path (path); + + g_autofree char *contents = NULL; + gsize len; + g_assert_true (g_file_load_contents (sig_file, NULL, &contents, + &len, NULL, &local_error)); + g_assert_no_error (local_error); + + char *cur = contents; + while (len > 0) + { + ssize_t written = gpgme_data_write (signature_buffer, cur, len); + if (written == -1) + assert_no_gpg_error (gpgme_error_from_syserror (), path); + cur += written; + len -= written; + } + } + + gpgme_data_seek (signature_buffer, 0, SEEK_SET); + } gpg_error = gpgme_op_verify (result->context, signature_buffer, data_buffer, NULL); @@ -123,7 +155,7 @@ test_signature_lookup (TestFixture *fixture, gconstpointer user_data) { /* Checking the signature with the revoked key for this case. */ - guint expected_signature_index = GPOINTER_TO_UINT (user_data); + guint expected_signature_index = 2; /* Lowercase letters to ensure OstreeGpgVerifyResult handles it. */ const char *fingerprint = "68dcc2db4bec5811c2573590bd9d2a44b7f541a6"; @@ -215,7 +247,7 @@ static void test_valid_signature (TestFixture *fixture, gconstpointer user_data) { - guint signature_index = GPOINTER_TO_UINT (user_data); + guint signature_index = 0; g_autoptr(GVariant) tuple = NULL; gboolean valid; gboolean sig_expired; @@ -249,7 +281,7 @@ static void test_expired_key (TestFixture *fixture, gconstpointer user_data) { - guint signature_index = GPOINTER_TO_UINT (user_data); + guint signature_index = 1; g_autoptr(GVariant) tuple = NULL; gboolean valid; gboolean sig_expired; @@ -283,7 +315,7 @@ static void test_revoked_key (TestFixture *fixture, gconstpointer user_data) { - guint signature_index = GPOINTER_TO_UINT (user_data); + guint signature_index = 2; g_autoptr(GVariant) tuple = NULL; gboolean valid; gboolean sig_expired; @@ -317,7 +349,7 @@ static void test_missing_key (TestFixture *fixture, gconstpointer user_data) { - guint signature_index = GPOINTER_TO_UINT (user_data); + guint signature_index = 3; g_autoptr(GVariant) tuple = NULL; gboolean valid; gboolean sig_expired; @@ -351,7 +383,7 @@ static void test_expired_signature (TestFixture *fixture, gconstpointer user_data) { - guint signature_index = GPOINTER_TO_UINT (user_data); + guint signature_index = 4; g_autoptr(GVariant) tuple = NULL; gboolean valid; gboolean sig_expired; @@ -397,7 +429,7 @@ main (int argc, char **argv) g_test_add ("/gpg-verify-result/signature-lookup", TestFixture, - GINT_TO_POINTER (2), + NULL, test_fixture_setup, test_signature_lookup, test_fixture_teardown); @@ -411,35 +443,35 @@ main (int argc, char **argv) g_test_add ("/gpg-verify-result/valid-signature", TestFixture, - GINT_TO_POINTER (0), /* signature index */ + NULL, test_fixture_setup, test_valid_signature, test_fixture_teardown); g_test_add ("/gpg-verify-result/expired-key", TestFixture, - GINT_TO_POINTER (1), /* signature index */ + NULL, test_fixture_setup, test_expired_key, test_fixture_teardown); g_test_add ("/gpg-verify-result/revoked-key", TestFixture, - GINT_TO_POINTER (2), /* signature index */ + NULL, test_fixture_setup, test_revoked_key, test_fixture_teardown); g_test_add ("/gpg-verify-result/missing-key", TestFixture, - GINT_TO_POINTER (3), /* signature index */ + NULL, test_fixture_setup, test_missing_key, test_fixture_teardown); g_test_add ("/gpg-verify-result/expired-signature", TestFixture, - GINT_TO_POINTER (4), /* signature index */ + NULL, test_fixture_setup, test_expired_signature, test_fixture_teardown); From 0fbfc0b2079f32b919ae6b804bda40332e03b618 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Sat, 15 Jun 2019 09:56:44 -0500 Subject: [PATCH 30/49] lib/gpg: Add more specific OstreeGpgError codes Currently `ostree_gpg_verify_result_require_valid_signature` always returns an error that the key used for the signature is missing from the keyring. However, all that's been determined is that there are no valid signatures. The error could also be from an expired signature, an expired key, a revoked key or an invalid signature. Provide values for these missing errors and return them from `ostree_gpg_verify_result_require_valid_signature`. The description of each result is appended to the error message, but since the result can contain more than one signature but only a single error can be returned, the status of the last signature is used for the error code. See the comment for rationale. Related: flatpak/flatpak#1450 --- src/libostree/ostree-gpg-verify-result.c | 54 +++++++++++++++++++++++- src/libostree/ostree-gpg-verify-result.h | 8 ++++ tests/test-pull-summary-sigs.sh | 2 +- tests/test-remote-gpg-import.sh | 6 +-- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index 63c3394a..67270c82 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -769,8 +769,58 @@ ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult *result, if (ostree_gpg_verify_result_count_valid (result) == 0) { - g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_MISSING_KEY, - "GPG signatures found, but none are in trusted keyring"); + /* + * Join the description of each failed signature for the error message. + * Only one error code can be returned, so if there was more than one + * signature, use the error of the last one under the assumption that + * it's the most recent and hopefully most likely to be made with a + * valid key. + */ + gint code = OSTREE_GPG_ERROR_NO_SIGNATURE; + g_autoptr(GString) buffer = g_string_sized_new (256); + guint nsigs = ostree_gpg_verify_result_count_all (result); + + if (nsigs == 0) + /* In case an empty result was passed in */ + g_string_append (buffer, "No GPG signatures found"); + else + { + for (int i = nsigs - 1; i >= 0; i--) + { + g_autoptr(GVariant) info = ostree_gpg_verify_result_get_all (result, i); + ostree_gpg_verify_result_describe_variant (info, buffer, "", + OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT); + + if (i == nsigs - 1) + { + gboolean key_missing, key_revoked, key_expired, sig_expired; + g_variant_get_child (info, OSTREE_GPG_SIGNATURE_ATTR_KEY_MISSING, + "b", &key_missing); + g_variant_get_child (info, OSTREE_GPG_SIGNATURE_ATTR_KEY_REVOKED, + "b", &key_revoked); + g_variant_get_child (info, OSTREE_GPG_SIGNATURE_ATTR_KEY_EXPIRED, + "b", &key_expired); + g_variant_get_child (info, OSTREE_GPG_SIGNATURE_ATTR_SIG_EXPIRED, + "b", &sig_expired); + + if (key_missing) + code = OSTREE_GPG_ERROR_MISSING_KEY; + else if (key_revoked) + code = OSTREE_GPG_ERROR_REVOKED_KEY; + else if (key_expired) + code = OSTREE_GPG_ERROR_EXPIRED_KEY; + else if (sig_expired) + code = OSTREE_GPG_ERROR_EXPIRED_SIGNATURE; + else + /* Assume any other issue is a bad signature */ + code = OSTREE_GPG_ERROR_INVALID_SIGNATURE; + } + } + } + + /* Strip any trailing newlines */ + g_strchomp (buffer->str); + g_set_error_literal (error, OSTREE_GPG_ERROR, code, buffer->str); return FALSE; } diff --git a/src/libostree/ostree-gpg-verify-result.h b/src/libostree/ostree-gpg-verify-result.h index 7c71ecdc..f71ab981 100644 --- a/src/libostree/ostree-gpg-verify-result.h +++ b/src/libostree/ostree-gpg-verify-result.h @@ -159,6 +159,11 @@ gboolean ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult * @OSTREE_GPG_ERROR_NO_SIGNATURE: A signature was expected, but not found. * @OSTREE_GPG_ERROR_INVALID_SIGNATURE: A signature was malformed. * @OSTREE_GPG_ERROR_MISSING_KEY: A signature was found, but was created with a key not in the configured keyrings. + * @OSTREE_GPG_ERROR_EXPIRED_SIGNATURE: A signature was expired. Since: 2019.7. + * @OSTREE_GPG_ERROR_EXPIRED_KEY: A signature was found, but the key used to + * sign it has expired. Since: 2019.7. + * @OSTREE_GPG_ERROR_REVOKED_KEY: A signature was found, but the key used to + * sign it has been revoked. Since: 2019.7. * * Errors returned by signature creation and verification operations in OSTree. * These may be returned by any API which creates or verifies signatures. @@ -169,6 +174,9 @@ typedef enum { OSTREE_GPG_ERROR_NO_SIGNATURE = 0, OSTREE_GPG_ERROR_INVALID_SIGNATURE, OSTREE_GPG_ERROR_MISSING_KEY, + OSTREE_GPG_ERROR_EXPIRED_SIGNATURE, + OSTREE_GPG_ERROR_EXPIRED_KEY, + OSTREE_GPG_ERROR_REVOKED_KEY, } OstreeGpgError; /** diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index 821ae953..401e88c9 100755 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -189,7 +189,7 @@ cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with old summary" fi -assert_file_has_content err.txt "none are in trusted keyring" +assert_file_has_content err.txt "BAD signature" assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index 4d73fa11..b8673852 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -163,7 +163,7 @@ ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key1.asc,${test_tmp if ${OSTREE} pull R8:main 2>err.txt; then assert_not_reached "Unexpectedly succeeded at pulling with different key" fi -assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring" +assert_file_has_content err.txt "public key not found" # Test gpgkeypath success with directory containing a valid key ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/ R9 $(cat httpd-address)/ostree/gnomerepo @@ -243,7 +243,7 @@ ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key2.asc R6 $(cat h if ${OSTREE} pull R6:main 2>err.txt; then assert_not_reached "Unexpectedly succeeded at pulling with different key" fi -assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring" +assert_file_has_content err.txt "public key not found" echo "ok" @@ -269,7 +269,7 @@ newrev=$(${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo rev-par if ${OSTREE} pull --require-static-deltas R1:main 2>err.txt; then assert_not_reached "Unexpectedly succeeded at pulling commit signed with untrusted key" fi -assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring" +assert_file_has_content err.txt "public key not found" echo "ok gpg untrusted signed commit for delta upgrades" From 01da2371c5451edc184915fa50b051fb589a157e Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 21 Jun 2019 06:16:13 -0500 Subject: [PATCH 31/49] tests/gpg: Test ostree_gpg_verify_result_require_valid_signature Add explicit tests for `ostree_gpg_verify_result_require_valid_signature` in addition to the implicit tests via `ostree pull` and others. This allows checking the error code raised. --- tests/test-gpg-verify-result.c | 136 +++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/tests/test-gpg-verify-result.c b/tests/test-gpg-verify-result.c index 1b4e54c1..5ae129b9 100644 --- a/tests/test-gpg-verify-result.c +++ b/tests/test-gpg-verify-result.c @@ -36,6 +36,18 @@ } \ } G_STMT_END +#define assert_str_contains(s1, s2) \ + G_STMT_START { \ + const char *__s1 = (s1), *__s2 = (s2); \ + if (strstr (__s1, __s2) == NULL) { \ + g_autoptr(GString) string = g_string_new ("assertion failed (" #s1 " contains " #s2 "): "); \ + g_autofree char *__es1 = g_strescape (__s1, NULL); \ + g_autofree char *__es2 = g_strescape (__s2, NULL); \ + g_string_append_printf (string, "(\"%s\", \"%s\")", __es1, __es2); \ + g_assertion_message (G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, string->str); \ + } \ + } G_STMT_END + typedef struct { OstreeGpgVerifyResult *result; } TestFixture; @@ -413,6 +425,83 @@ test_expired_signature (TestFixture *fixture, g_assert_cmpint (key_exp_timestamp, ==, 0); } +static void +test_require_valid_signature (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_true (res); + g_assert_no_error (error); +} + +static void +test_require_valid_signature_expired_key (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_false (res); + g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_EXPIRED_KEY); + assert_str_contains (error->message, "Key expired"); +} + +static void +test_require_valid_signature_revoked_key (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_false (res); + g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_REVOKED_KEY); + assert_str_contains (error->message, "Key revoked"); +} + +static void +test_require_valid_signature_missing_key (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_false (res); + g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_MISSING_KEY); + assert_str_contains (error->message, "public key not found"); +} + +static void +test_require_valid_signature_expired_signature (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_false (res); + g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_EXPIRED_SIGNATURE); + assert_str_contains (error->message, "Signature expired"); +} + +static void +test_require_valid_signature_expired_missing_key (TestFixture *fixture, + gconstpointer user_data) +{ + GError *error = NULL; + gboolean res = ostree_gpg_verify_result_require_valid_signature (fixture->result, + &error); + g_assert_false (res); + + /* + * The error will be for the last signature, which is for a missing key, but + * the message should show both issues. + */ + g_assert_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_MISSING_KEY); + assert_str_contains (error->message, "Key expired"); + assert_str_contains (error->message, "public key not found"); +} + int main (int argc, char **argv) { @@ -476,5 +565,52 @@ main (int argc, char **argv) test_expired_signature, test_fixture_teardown); + g_test_add ("/gpg-verify-result/require-valid-signature", + TestFixture, + NULL, + test_fixture_setup, + test_require_valid_signature, + test_fixture_teardown); + + const char *expired_key_files[] = { "lgpl2.sig1", NULL }; + g_test_add ("/gpg-verify-result/require-valid-signature-expired-key", + TestFixture, + expired_key_files, + test_fixture_setup, + test_require_valid_signature_expired_key, + test_fixture_teardown); + + const char *revoked_key_files[] = { "lgpl2.sig2", NULL }; + g_test_add ("/gpg-verify-result/require-valid-signature-revoked-key", + TestFixture, + revoked_key_files, + test_fixture_setup, + test_require_valid_signature_revoked_key, + test_fixture_teardown); + + const char *missing_key_files[] = { "lgpl2.sig3", NULL }; + g_test_add ("/gpg-verify-result/require-valid-signature-missing-key", + TestFixture, + missing_key_files, + test_fixture_setup, + test_require_valid_signature_missing_key, + test_fixture_teardown); + + const char *expired_signature_files[] = { "lgpl2.sig4", NULL }; + g_test_add ("/gpg-verify-result/require-valid-signature-expired-signature", + TestFixture, + expired_signature_files, + test_fixture_setup, + test_require_valid_signature_expired_signature, + test_fixture_teardown); + + const char *expired_missing_key_files[] = { "lgpl2.sig1", "lgpl2.sig3", NULL }; + g_test_add ("/gpg-verify-result/require-valid-signature-expired-missing-key", + TestFixture, + expired_missing_key_files, + test_fixture_setup, + test_require_valid_signature_expired_missing_key, + test_fixture_teardown); + return g_test_run (); } From b81a6b4ab2186b9521b9144b7247f2a26d9c675c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 21 Jun 2019 06:59:56 -0500 Subject: [PATCH 32/49] tests/gpg: Add tests for importing updated remote GPG keys This checks whether gpg-import will properly update the keyring for a key that already exists. In particular, we check that changing the key expiration time or revoking it results in commit verification failure after re-importing the keys. --- tests/test-remote-gpg-import.sh | 61 ++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index b8673852..cf13b499 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -28,7 +28,12 @@ unset OSTREE_GPG_HOME setup_fake_remote_repo1 "archive" -echo "1..4" +# Some tests require an appropriate gpg +num_non_gpg_tests=4 +num_gpg_tests=2 +num_tests=$((num_non_gpg_tests + num_gpg_tests)) + +echo "1..${num_tests}" cd ${test_tmpdir} mkdir repo @@ -280,3 +285,57 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u --gpg ${OSTREE} pull --require-static-deltas R1:main echo "ok gpg trusted signed commit for delta upgrades" + +# Run some more tests if an appropriate gpg is available +GPG=$(which_gpg) +if [ -z "${GPG}" ]; then + # Print a skip message per skipped test + for (( i = 0; i < num_gpg_tests; i++ )); do + echo "ok # SKIP this test requires gpg" + done +else + # Create a commit signed with keyid 1 + echo $(date) > workdir/testfile-for-key-mangling + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main --gpg-sign ${TEST_GPG_KEYID_1} --gpg-homedir ${test_tmpdir}/gpghome workdir + + # Re-add the remote + ${OSTREE} remote delete R1 + ${OSTREE} remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc R1 $(cat httpd-address)/ostree/gnomerepo | grep -o 'Imported [[:digit:]] GPG key' > result + assert_file_has_content result 'Imported 1 GPG key' + + # Expire key 1, wait for it to be expired and import the expired key. Only + # new keys are reported. + ${GPG} --homedir=${test_tmpdir}/gpghome --quick-set-expire ${TEST_GPG_KEYFPR_1} seconds=1 + sleep 2 + ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1expired.asc + ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1expired.asc R1 | grep -o 'Imported [[:digit:]] GPG key' > result + assert_file_has_content result 'Imported 0 GPG key' + + # Pulling should fail since the key is expired + rm repo/refs/remotes/* -rf + ${OSTREE} prune --refs-only + if ${OSTREE} pull R1:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling commit signed with expired key" + fi + assert_file_has_content err.txt "Key expired" + + echo "ok imported expired key" + + # Unexpire keyid 1 and revoke it. Revoking is done by importing the + # pre-generated revocation certificate. + ${GPG} --homedir=${test_tmpdir}/gpghome --quick-set-expire ${TEST_GPG_KEYFPR_1} seconds=0 + ${GPG} --homedir=${TEST_GPG_KEYHOME} --import ${TEST_GPG_KEYHOME}/revocations/key1.rev + ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1revoked.asc + ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1revoked.asc R1 | grep -o 'Imported [[:digit:]] GPG key' > result + assert_file_has_content result 'Imported 0 GPG key' + + # Pulling should fail since the key is revoked + rm repo/refs/remotes/* -rf + ${OSTREE} prune --refs-only + if ${OSTREE} pull R1:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling commit signed with revoked key" + fi + assert_file_has_content err.txt "Key revoked" + + echo "ok imported revoked key" +fi From 68a11d4eeb3800537292f2538536b6230259f5d2 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 21 Jan 2020 15:47:01 -0700 Subject: [PATCH 33/49] ci/flatpak: Patch GPG error assertions from OSTree Some of the flatpak tests assert on GPG error strings that come from OSTree. Those are being changed here, so patch the cloned flatpak 1.4.1 to accommodate the new error strings. When this work lands, I'll send a patch upstream to flatpak that will eventually trickle back here in a tagged build. --- ci/flatpak-1.4.1-ostree-gpg-errors.patch | 65 ++++++++++++++++++++++++ ci/flatpak.sh | 6 +++ 2 files changed, 71 insertions(+) create mode 100644 ci/flatpak-1.4.1-ostree-gpg-errors.patch diff --git a/ci/flatpak-1.4.1-ostree-gpg-errors.patch b/ci/flatpak-1.4.1-ostree-gpg-errors.patch new file mode 100644 index 00000000..6b10c584 --- /dev/null +++ b/ci/flatpak-1.4.1-ostree-gpg-errors.patch @@ -0,0 +1,65 @@ +From 8e649d094e9dd91adbb430015b2621c66e086df7 Mon Sep 17 00:00:00 2001 +From: Dan Nicholson +Date: Tue, 21 Jan 2020 15:32:27 -0700 +Subject: [PATCH] tests: Accommodate new OSTree GPG error strings + +Recently OSTree has been updated to provide proper error strings when +validating GPG signatures instead of a single generic string[1]. Allow +either in the tests so they work against new or old ostree. + +1. https://github.com/ostreedev/ostree/pull/1877 +--- + tests/test-p2p-security.sh | 2 +- + tests/test-repo.sh | 8 ++++---- + 2 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/tests/test-p2p-security.sh b/tests/test-p2p-security.sh +index db929dd4..9b0ca1d0 100644 +--- a/tests/test-p2p-security.sh ++++ b/tests/test-p2p-security.sh +@@ -53,7 +53,7 @@ GPGARGS="${FL_GPGARGS2}" make_updated_app test-impostor org.test.Collection + if G_MESSAGES_DEBUG=all ${FLATPAK} ${U} update -y org.test.Hello >failed-p2p-update-log; then + assert_not_reached "Update of org.test.Hello was successful despite malicious commit" + fi +-assert_file_has_content failed-p2p-update-log "GPG signatures found, but none are in trusted keyring" ++assert_file_has_content failed-p2p-update-log "\(GPG signatures found, but none are in trusted keyring\|public key not found\)" + + COMMIT_AFTER_FAILED_UPDATE=$(${FLATPAK} ${U} info -c org.test.Hello) + if [ "x${INITIAL_COMMIT}" != "x${COMMIT_AFTER_FAILED_UPDATE}" ]; then +diff --git a/tests/test-repo.sh b/tests/test-repo.sh +index 01ca6a94..14cb1179 100644 +--- a/tests/test-repo.sh ++++ b/tests/test-repo.sh +@@ -183,25 +183,25 @@ ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello + if ${FLATPAK} ${U} install -y test-missing-gpg-repo org.test.Platform 2> install-error-log; then + assert_not_reached "Should not be able to install with missing gpg key" + fi +-assert_file_has_content install-error-log "GPG signatures found, but none are in trusted keyring" ++assert_file_has_content install-error-log "\(GPG signatures found, but none are in trusted keyring\|public key not found\)" + + + if ${FLATPAK} ${U} install test-missing-gpg-repo org.test.Hello 2> install-error-log; then + assert_not_reached "Should not be able to install with missing gpg key" + fi +-assert_file_has_content install-error-log "GPG signatures found, but none are in trusted keyring" ++assert_file_has_content install-error-log "\(GPG signatures found, but none are in trusted keyring\|public key not found\)" + + echo "ok fail with missing gpg key" + + if ${FLATPAK} ${U} install test-wrong-gpg-repo org.test.Platform 2> install-error-log; then + assert_not_reached "Should not be able to install with wrong gpg key" + fi +-assert_file_has_content install-error-log "GPG signatures found, but none are in trusted keyring" ++assert_file_has_content install-error-log "\(GPG signatures found, but none are in trusted keyring\|public key not found\)" + + if ${FLATPAK} ${U} install test-wrong-gpg-repo org.test.Hello 2> install-error-log; then + assert_not_reached "Should not be able to install with wrong gpg key" + fi +-assert_file_has_content install-error-log "GPG signatures found, but none are in trusted keyring" ++assert_file_has_content install-error-log "\(GPG signatures found, but none are in trusted keyring\|public key not found\)" + + echo "ok fail with wrong gpg key" + +-- +2.20.1 + diff --git a/ci/flatpak.sh b/ci/flatpak.sh index fd76b6fd..989b1235 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -22,6 +22,12 @@ tmpd=$(mktemp -d) cd ${tmpd} git clone --recursive --depth=1 -b ${FLATPAK_TAG} https://github.com/flatpak/flatpak cd ${tmpd}/flatpak + +# Some of flatpak's tests assert GPG error strings from ostree, but +# those have been changed. Patch the test assertions until this can get +# into a tagged flatpak. +git apply ${codedir}/ci/flatpak-1.4.1-ostree-gpg-errors.patch + # This is a copy of flatpak/ci/build.sh, but we can't use that as we want to install # our built ostree over it. pkg_install sudo which attr fuse bison \ From fbf5a94e0aa2d55203ac03fa941a28ce9f6ba071 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 18 Oct 2019 10:33:02 -0600 Subject: [PATCH 34/49] ostree/trivial-httpd: Fix --autoexit with --daemonize and --log-file When --autoexit is used with --daemonize and --log-file, the program never exits when the root directory is deleted. I believe what happens is that g_file_new_for_path triggers the glib worker context to be started to talk to GVfs. Once the program forks, the parent exits and the thread iterating the worker context is gone. The file monitor then never receives any events because the inotify helper also runs from the worker context. Move the fork earlier just after parsing and validating the command line arguments. In order to handle setup errors in the child, a pipe is opened and the parents waits until the child writes a status byte to it. If the byte is 0, the parent considers the child setup successful and exits and the child carries on as a daemon. Notably, the child doesn't reopen stderr to /dev/null until after this so that it can send error messages there. Fixes: #1941 --- src/ostree/ostree-trivial-httpd.c | 149 +++++++++++++++++++++++------- 1 file changed, 114 insertions(+), 35 deletions(-) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 5d3a004e..e5dfc320 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -507,6 +507,7 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) const char *dirpath; OtTrivialHttpd appstruct = { 0, }; OtTrivialHttpd *app = &appstruct; + int pipefd[2] = { -1, -1 }; glnx_unref_object SoupServer *server = NULL; g_autoptr(GFileMonitor) dirmon = NULL; @@ -540,17 +541,86 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) goto out; } + if (opt_daemonize && (g_strcmp0 (opt_log, "-") == 0)) + { + ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error); + goto out; + } + + /* Fork early before glib sets up its worker context and thread since they'll + * be gone once the parent exits. The parent waits on a pipe with the child to + * handle setup errors. The child writes a 0 when setup is successful and a 1 + * otherwise. + */ + if (opt_daemonize) + { + if (pipe (pipefd) == -1) + { + glnx_set_error_from_errno (error); + goto out; + } + + pid_t pid = fork(); + if (pid == -1) + { + glnx_set_error_from_errno (error); + goto out; + } + else if (pid > 0) + { + /* Parent, read the child status from the pipe */ + glnx_close_fd (&pipefd[1]); + guint8 buf; + int res = TEMP_FAILURE_RETRY (read (pipefd[0], &buf, 1)); + if (res < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + else if (res == 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Child process closed pipe without writing status"); + goto out; + } + + g_debug ("Read %u from child", buf); + if (buf > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Child process failed during setup"); + goto out; + } + glnx_close_fd (&pipefd[0]); + + ret = TRUE; + goto out; + } + + /* Child, continue */ + glnx_close_fd (&pipefd[0]); + } + else + { + /* Since we're used for testing purposes, let's just do this by + * default. This ensures we exit when our parent does. + */ + if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0) + { + if (errno != ENOSYS) + { + glnx_set_error_from_errno (error); + goto out; + } + } + } + if (opt_log) { GOutputStream *stream = NULL; if (g_strcmp0 (opt_log, "-") == 0) { - if (opt_daemonize) - { - ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error); - goto out; - } stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE)); } else @@ -625,45 +695,39 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) #if !SOUP_CHECK_VERSION(2, 48, 0) soup_server_run_async (server); #endif - + if (opt_daemonize) { - pid_t pid = fork(); - if (pid == -1) + /* Write back a 0 to the pipe to indicate setup was successful. */ + guint8 buf = 0; + g_debug ("Writing %u to parent", buf); + if (TEMP_FAILURE_RETRY (write (pipefd[1], &buf, 1)) == -1) { - int errsv = errno; - g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (errsv), - g_strerror (errsv)); + glnx_set_error_from_errno (error); goto out; } - else if (pid > 0) - { - ret = TRUE; - goto out; - } - /* Child, continue */ + glnx_close_fd (&pipefd[1]); + if (setsid () < 0) - err (1, "setsid"); + { + glnx_set_prefix_error_from_errno (error, "%s", "setsid: "); + goto out; + } /* Daemonising: close stdout/stderr so $() et al work on us */ if (freopen("/dev/null", "r", stdin) == NULL) - err (1, "freopen"); - if (freopen("/dev/null", "w", stdout) == NULL) - err (1, "freopen"); - if (freopen("/dev/null", "w", stderr) == NULL) - err (1, "freopen"); - } - else - { - /* Since we're used for testing purposes, let's just do this by - * default. This ensures we exit when our parent does. - */ - if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0) { - if (errno != ENOSYS) - { - glnx_set_error_from_errno (error); - goto out; - } + glnx_set_prefix_error_from_errno (error, "%s", "freopen: "); + goto out; + } + if (freopen("/dev/null", "w", stdout) == NULL) + { + glnx_set_prefix_error_from_errno (error, "%s", "freopen: "); + goto out; + } + if (freopen("/dev/null", "w", stderr) == NULL) + { + glnx_set_prefix_error_from_errno (error, "%s", "freopen: "); + goto out; } } @@ -699,6 +763,21 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error) ret = TRUE; out: + if (pipefd[0] >= 0) + { + /* Read end in the parent. This should only be open on errors. */ + g_assert_false (ret); + glnx_close_fd (&pipefd[0]); + } + if (pipefd[1] >= 0) + { + /* Write end in the child. This should only be open on errors. */ + g_assert_false (ret); + guint8 buf = 1; + g_debug ("Writing %u to parent", buf); + (void) TEMP_FAILURE_RETRY (write (pipefd[1], &buf, 1)); + glnx_close_fd (&pipefd[1]); + } if (app->root_dfd != -1) (void) close (app->root_dfd); g_clear_object (&app->log); From 11ad68647a539344d81eda92cf5ec24d8c2354b1 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 18 Oct 2019 10:50:39 -0600 Subject: [PATCH 35/49] ostree/trivial-httpd: Add log message for autoexit This is useful when checking if the daemon actually exited since we don't store the child PID anywhere. --- src/ostree/ostree-trivial-httpd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index e5dfc320..a38abbea 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -494,6 +494,7 @@ on_dir_changed (GFileMonitor *mon, if (event == G_FILE_MONITOR_EVENT_DELETED) { + httpd_log (self, "root directory removed, exiting\n"); self->running = FALSE; g_main_context_wakeup (NULL); } From 562b60022b81b5ffcef0b8f8497bf410b33542e0 Mon Sep 17 00:00:00 2001 From: Alex Kiernan Date: Tue, 21 Jan 2020 12:37:52 +0000 Subject: [PATCH 36/49] build: fix systemd feature advertisement 17db0f15a798 ("configure: add option for libsystemd") exposed --without-libsystemd to allow systemd to be disabled even if the systemd pkgconfig script was present, introducing a new variable with_libsystemd; there are now three, almost identical variables: - with_libsystemd [yes, no, maybe] - controlled by --without-libsystemd, resolved into yes/no by the initial checks - have_libsystemd [yes, no, ] - only set if with_libsystemd is yes/maybe, otherwise undefined - with_systemd [yes, ] - yes if have_systemd is yes, otherwise undefined with_systemd is the earliest variable and was previously set by a set of checks for dracut and mkinitcpio. These checks were changed for a systemd check in 9e2763106be0 ("lib: Use sd_journal directly (optionally)"). This commit also introduced BUILDOPT_LIBSYSTEMD, which will always match BUILDOPT_SYSTEMD. Fix the confusion by removing with_systemd which will always be yes when with_libsystemd=yes, or undefined if with_libsystemd=no. We can ignore the with_libsystemd=maybe case because it will always be resolved into yes/no before with_systemd is set. And replace all uses of BUILDOPT_SYSTEMD with BUILDOPT_LIBSYSTEMD, since they again always match. This fixes both the advertised features and the summary output when systemd is disabled by using with_libsystemd which is always defined. Signed-off-by: Alex Kiernan Fixes: 5c62a7e4d0a5 ("build: Expose systemd in OSTREE_FEATURES") Fixes: 17db0f15a798 ("configure: add option for libsystemd") Supersedes: #1992 --- Makefile-libostree.am | 2 +- configure.ac | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 4b412fec..a7e7e123 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -214,7 +214,7 @@ libostree_1_la_CFLAGS += $(OT_DEP_AVAHI_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_AVAHI_LIBS) endif -if BUILDOPT_LIBSYSTEMD +if BUILDOPT_SYSTEMD libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) endif diff --git a/configure.ac b/configure.ac index 429239f4..98725fc4 100644 --- a/configure.ac +++ b/configure.ac @@ -509,10 +509,8 @@ AS_IF([ test x$with_libsystemd != xno ], [ with_libsystemd=no ]) ], [ with_libsystemd=no ]) -AM_CONDITIONAL(BUILDOPT_LIBSYSTEMD, test $with_libsystemd != no) -AS_IF([test "x$have_libsystemd" = "xyes"], [ - with_systemd=yes +AS_IF([test "x$with_libsystemd" = "xyes"], [ AC_ARG_WITH([systemdsystemunitdir], AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files]), [], @@ -528,12 +526,12 @@ AS_IF([test "x$have_libsystemd" = "xyes"], [ AC_SUBST([systemdsystemgeneratordir], [$with_systemdsystemgeneratordir]) ]) ]) -AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_systemd = xyes) +AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_libsystemd = xyes) dnl If we have both, we use the "new /var" model with ostree-system-generator -AM_CONDITIONAL(BUILDOPT_SYSTEMD_AND_LIBMOUNT,[test x$with_systemd = xyes && test x$with_libmount = xyes]) +AM_CONDITIONAL(BUILDOPT_SYSTEMD_AND_LIBMOUNT,[test x$with_libsystemd = xyes && test x$with_libmount = xyes]) AM_COND_IF(BUILDOPT_SYSTEMD_AND_LIBMOUNT, AC_DEFINE([BUILDOPT_LIBSYSTEMD_AND_LIBMOUNT], 1, [Define if systemd and libmount])) -if test x$with_systemd != xno; then OSTREE_FEATURES="$OSTREE_FEATURES systemd"; fi +if test x$with_libsystemd = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES systemd"; fi AC_ARG_WITH(builtin-grub2-mkconfig, AS_HELP_STRING([--with-builtin-grub2-mkconfig], @@ -621,7 +619,7 @@ echo " \"ostree trivial-httpd\": $enable_trivial_httpd_cmdline SELinux: $with_selinux cryptographic checksums: $with_crypto - systemd: $have_libsystemd + systemd: $with_libsystemd libmount: $with_libmount libarchive (parse tar files directly): $with_libarchive static deltas: yes (always enabled now) From 58fa579b1be19104025b1f71b4e52a255cc2a50b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 25 Oct 2019 22:07:44 +0000 Subject: [PATCH 37/49] Initial fs-verity support Using fs-verity is natural for OSTree because it's file-based, as opposed to block based (like dm-verity). This only covers files - not symlinks or directories. And we clearly need to have integrity for the deployment directories at least. Also, what we likely need is an API that supports signing files as they're committed. So making this truly secure would need a lot more work. Nevertheless, I think it's time to start experimenting with it. Among other things, it does *finally* add an API that makes files immutable, which will help against some accidental damage. This is basic enablement work that is being driven by Fedora CoreOS; see also https://github.com/coreos/coreos-assembler/pull/876 --- configure.ac | 5 ++ src/libostree/ostree-repo-commit.c | 114 ++++++++++++++++++++++++++ src/libostree/ostree-repo-private.h | 20 +++++ src/libostree/ostree-repo.c | 30 +++++++ src/libostree/ostree-sysroot-deploy.c | 99 +++++++++++++++------- 5 files changed, 237 insertions(+), 31 deletions(-) diff --git a/configure.ac b/configure.ac index 98725fc4..a28172a1 100644 --- a/configure.ac +++ b/configure.ac @@ -246,6 +246,10 @@ LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" # What's in RHEL7.2. FUSE_DEPENDENCY="fuse >= 2.9.2" +AC_CHECK_HEADERS([linux/fsverity.h]) +AS_IF([test x$ac_cv_header_linux_fsverity_h = xyes ], + [OSTREE_FEATURES="$OSTREE_FEATURES ex-fsverity"]) + # check for gtk-doc m4_ifdef([GTK_DOC_CHECK], [ GTK_DOC_CHECK([1.15], [--flavour no-tmpl]) @@ -618,6 +622,7 @@ echo " HTTP backend: $fetcher_backend \"ostree trivial-httpd\": $enable_trivial_httpd_cmdline SELinux: $with_selinux + fs-verity: $ac_cv_header_linux_fsverity_h cryptographic checksums: $with_crypto systemd: $with_libsystemd libmount: $with_libmount diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8c5d9411..38fc6ef1 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -32,6 +32,10 @@ #include #include #include +#include +#ifdef HAVE_LINUX_FSVERITY_H +#include +#endif #include "otutil.h" #include "ostree.h" @@ -168,6 +172,113 @@ ot_security_smack_reset_fd (int fd) #endif } +/* Wrapper around the fsverity ioctl, compressing the result to + * "success, unsupported or error". This is used for /boot where + * we enable verity if supported. + * */ +gboolean +_ostree_tmpf_fsverity_core (GLnxTmpfile *tmpf, + _OstreeFeatureSupport fsverity_requested, + gboolean *supported, + GError **error) +{ + /* Set this by default to simplify the code below */ + if (supported) + *supported = FALSE; + + if (fsverity_requested == _OSTREE_FEATURE_NO) + return TRUE; + +#ifdef HAVE_LINUX_FSVERITY_H + GLNX_AUTO_PREFIX_ERROR ("fsverity", error); + + /* fs-verity requires a read-only file descriptor */ + if (!glnx_tmpfile_reopen_rdonly (tmpf, error)) + return FALSE; + + struct fsverity_enable_arg arg = { 0, }; + arg.version = 1; + arg.hash_algorithm = FS_VERITY_HASH_ALG_SHA256; /* TODO configurable? */ + arg.block_size = 4096; /* FIXME query */ + arg.salt_size = 0; /* TODO store salt in ostree repo config */ + arg.salt_ptr = 0; + arg.sig_size = 0; /* We don't currently expect use of in-kernel signature verification */ + arg.sig_ptr = 0; + + if (ioctl (tmpf->fd, FS_IOC_ENABLE_VERITY, &arg) < 0) + { + switch (errno) + { + case ENOTTY: + case EOPNOTSUPP: + return TRUE; + default: + return glnx_throw_errno_prefix (error, "ioctl(FS_IOC_ENABLE_VERITY)"); + } + } + + if (supported) + *supported = TRUE; +#endif + return TRUE; +} + +/* Enable verity on a file, respecting the "wanted" and "supported" states. + * The main idea here is to optimize out pointlessly calling the ioctl() + * over and over in cases where it's not supported for the repo's filesystem, + * as well as to support "opportunistic" use (requested and if filesystem supports). + * */ +gboolean +_ostree_tmpf_fsverity (OstreeRepo *self, + GLnxTmpfile *tmpf, + GError **error) +{ +#ifdef HAVE_LINUX_FSVERITY_H + g_mutex_lock (&self->txn_lock); + _OstreeFeatureSupport fsverity_wanted = self->fs_verity_wanted; + _OstreeFeatureSupport fsverity_supported = self->fs_verity_supported; + g_mutex_unlock (&self->txn_lock); + + switch (fsverity_wanted) + { + case _OSTREE_FEATURE_YES: + { + if (fsverity_supported == _OSTREE_FEATURE_NO) + return glnx_throw (error, "fsverity required but filesystem does not support it"); + } + break; + case _OSTREE_FEATURE_MAYBE: + break; + case _OSTREE_FEATURE_NO: + return TRUE; + } + + gboolean supported = FALSE; + if (!_ostree_tmpf_fsverity_core (tmpf, fsverity_wanted, &supported, error)) + return FALSE; + + if (!supported) + { + if (G_UNLIKELY (fsverity_wanted == _OSTREE_FEATURE_YES)) + return glnx_throw (error, "fsverity required but filesystem does not support it"); + + /* If we got here, we must be trying "opportunistic" use of fs-verity */ + g_assert_cmpint (fsverity_wanted, ==, _OSTREE_FEATURE_MAYBE); + g_mutex_lock (&self->txn_lock); + self->fs_verity_supported = _OSTREE_FEATURE_NO; + g_mutex_unlock (&self->txn_lock); + return TRUE; + } + + g_mutex_lock (&self->txn_lock); + self->fs_verity_supported = _OSTREE_FEATURE_YES; + g_mutex_unlock (&self->txn_lock); +#else + g_assert_cmpint (self->fs_verity_wanted, !=, _OSTREE_FEATURE_YES); +#endif + return TRUE; +} + /* Given an O_TMPFILE regular file, link it into place. */ gboolean _ostree_repo_commit_tmpf_final (OstreeRepo *self, @@ -185,6 +296,9 @@ _ostree_repo_commit_tmpf_final (OstreeRepo *self, cancellable, error)) return FALSE; + if (!_ostree_tmpf_fsverity (self, tmpf, error)) + return FALSE; + if (!glnx_link_tmpfile_at (tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, dest_dfd, tmpbuf, error)) return FALSE; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b57ad799..3b629913 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -22,6 +22,7 @@ #pragma once #include +#include "config.h" #include "otutil.h" #include "ostree-ref.h" #include "ostree-repo.h" @@ -97,6 +98,12 @@ typedef struct { fsblkcnt_t max_blocks; } OstreeRepoTxn; +typedef enum { + _OSTREE_FEATURE_NO, + _OSTREE_FEATURE_MAYBE, + _OSTREE_FEATURE_YES, +} _OstreeFeatureSupport; + /** * OstreeRepo: * @@ -127,6 +134,8 @@ struct OstreeRepo { GMutex txn_lock; OstreeRepoTxn txn; gboolean txn_locked; + _OstreeFeatureSupport fs_verity_wanted; + _OstreeFeatureSupport fs_verity_supported; GMutex cache_lock; guint dirmeta_cache_refcount; @@ -471,4 +480,15 @@ OstreeRepoAutoLock * _ostree_repo_auto_lock_push (OstreeRepo *self, void _ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock); G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, _ostree_repo_auto_lock_cleanup) +gboolean +_ostree_tmpf_fsverity_core (GLnxTmpfile *tmpf, + _OstreeFeatureSupport fsverity_requested, + gboolean *supported, + GError **error); + +gboolean +_ostree_tmpf_fsverity (OstreeRepo *self, + GLnxTmpfile *tmpf, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index cff70d47..b6a5db69 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -31,6 +31,7 @@ #include "libglnx.h" #include "otutil.h" #include +#include #include "ostree-core-private.h" #include "ostree-sysroot-private.h" @@ -47,6 +48,7 @@ #include #include #include +#include #define REPO_LOCK_DISABLED (-2) #define REPO_LOCK_BLOCKING (-1) @@ -3033,6 +3035,34 @@ reload_core_config (OstreeRepo *self, } } + /* Currently experimental */ + static const char fsverity_key[] = "ex-fsverity"; + self->fs_verity_wanted = _OSTREE_FEATURE_NO; +#ifdef HAVE_LINUX_FSVERITY_H + self->fs_verity_supported = _OSTREE_FEATURE_MAYBE; +#else + self->fs_verity_supported = _OSTREE_FEATURE_NO; +#endif + gboolean fsverity_required = FALSE; + if (!ot_keyfile_get_boolean_with_default (self->config, fsverity_key, "required", + FALSE, &fsverity_required, error)) + return FALSE; + if (fsverity_required) + { + self->fs_verity_wanted = _OSTREE_FEATURE_YES; + if (self->fs_verity_supported == _OSTREE_FEATURE_NO) + return glnx_throw (error, "fsverity required, but libostree compiled without support"); + } + else + { + gboolean fsverity_opportunistic = FALSE; + if (!ot_keyfile_get_boolean_with_default (self->config, fsverity_key, "opportunistic", + FALSE, &fsverity_opportunistic, error)) + return FALSE; + if (fsverity_opportunistic) + self->fs_verity_wanted = _OSTREE_FEATURE_MAYBE; + } + { g_clear_pointer (&self->collection_id, g_free); if (!ot_keyfile_get_value_with_default (self->config, "core", "collection-id", diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 0dcda732..ee00c02c 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -41,6 +41,7 @@ #include "otutil.h" #include "ostree.h" +#include "ostree-repo-private.h" #include "ostree-sysroot-private.h" #include "ostree-sepolicy-private.h" #include "ostree-bootloader-zipl.h" @@ -104,7 +105,8 @@ sysroot_flags_to_copy_flags (GLnxFileCopyFlags defaults, * hardlink if we're on the same partition. */ static gboolean -install_into_boot (OstreeSePolicy *sepolicy, +install_into_boot (OstreeRepo *repo, + OstreeSePolicy *sepolicy, int src_dfd, const char *src_subpath, int dest_dfd, @@ -113,32 +115,67 @@ install_into_boot (OstreeSePolicy *sepolicy, GCancellable *cancellable, GError **error) { - if (linkat (src_dfd, src_subpath, dest_dfd, dest_subpath, 0) != 0) - { - if (G_IN_SET (errno, EMLINK, EXDEV)) - { - /* Be sure we relabel when copying the kernel, as in current - * e.g. Fedora it might be labeled module_object_t or usr_t, - * but policy may not allow other processes to read from that - * like kdump. - * See also https://github.com/fedora-selinux/selinux-policy/commit/747f4e6775d773ab74efae5aa37f3e5e7f0d4aca - * This means we also drop xattrs but...I doubt anyone uses - * non-SELinux xattrs for the kernel anyways aside from perhaps - * IMA but that's its own story. - */ - g_auto(OstreeSepolicyFsCreatecon) fscreatecon = { 0, }; - const char *boot_path = glnx_strjoina ("/boot/", glnx_basename (dest_subpath)); - if (!_ostree_sepolicy_preparefscreatecon (&fscreatecon, sepolicy, - boot_path, S_IFREG | 0644, - error)) - return FALSE; - return glnx_file_copy_at (src_dfd, src_subpath, NULL, dest_dfd, dest_subpath, - GLNX_FILE_COPY_NOXATTRS | GLNX_FILE_COPY_DATASYNC, - cancellable, error); - } - else - return glnx_throw_errno_prefix (error, "linkat(%s)", dest_subpath); - } + if (linkat (src_dfd, src_subpath, dest_dfd, dest_subpath, 0) == 0) + return TRUE; /* Note early return */ + if (!G_IN_SET (errno, EMLINK, EXDEV)) + return glnx_throw_errno_prefix (error, "linkat(%s)", dest_subpath); + + /* Otherwise, copy */ + struct stat src_stbuf; + if (!glnx_fstatat (src_dfd, src_subpath, &src_stbuf, AT_SYMLINK_NOFOLLOW, error)) + return FALSE; + + glnx_autofd int src_fd = -1; + if (!glnx_openat_rdonly (src_dfd, src_subpath, FALSE, &src_fd, error)) + return FALSE; + + /* Be sure we relabel when copying the kernel, as in current + * e.g. Fedora it might be labeled module_object_t or usr_t, + * but policy may not allow other processes to read from that + * like kdump. + * See also https://github.com/fedora-selinux/selinux-policy/commit/747f4e6775d773ab74efae5aa37f3e5e7f0d4aca + * This means we also drop xattrs but...I doubt anyone uses + * non-SELinux xattrs for the kernel anyways aside from perhaps + * IMA but that's its own story. + */ + g_auto(OstreeSepolicyFsCreatecon) fscreatecon = { 0, }; + const char *boot_path = glnx_strjoina ("/boot/", glnx_basename (dest_subpath)); + if (!_ostree_sepolicy_preparefscreatecon (&fscreatecon, sepolicy, + boot_path, S_IFREG | 0644, + error)) + return FALSE; + + g_auto(GLnxTmpfile) tmp_dest = { 0, }; + if (!glnx_open_tmpfile_linkable_at (dest_dfd, ".", O_WRONLY | O_CLOEXEC, + &tmp_dest, error)) + return FALSE; + + if (glnx_regfile_copy_bytes (src_fd, tmp_dest.fd, (off_t) -1) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); + + /* Kernel data should always be root-owned */ + if (fchown (tmp_dest.fd, src_stbuf.st_uid, src_stbuf.st_gid) != 0) + return glnx_throw_errno_prefix (error, "fchown"); + + if (fchmod (tmp_dest.fd, src_stbuf.st_mode & 07777) != 0) + return glnx_throw_errno_prefix (error, "fchmod"); + + if (fdatasync (tmp_dest.fd) < 0) + return glnx_throw_errno_prefix (error, "fdatasync"); + + /* Today we don't have a config flag to *require* verity on /boot, + * and at least for Fedora CoreOS we're not likely to do fsverity on + * /boot soon due to wanting to support mounting it from old Linux + * kernels. So change "required" to "maybe". + */ + _OstreeFeatureSupport boot_verity = _OSTREE_FEATURE_NO; + if (repo->fs_verity_wanted != _OSTREE_FEATURE_NO) + boot_verity = _OSTREE_FEATURE_MAYBE; + if (!_ostree_tmpf_fsverity_core (&tmp_dest, boot_verity, NULL, error)) + return FALSE; + + if (!glnx_link_tmpfile_at (&tmp_dest, GLNX_LINK_TMPFILE_NOREPLACE, dest_dfd, dest_subpath, error)) + return FALSE; return TRUE; } @@ -1666,7 +1703,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, return FALSE; if (errno == ENOENT) { - if (!install_into_boot (sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_srcpath, + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_srcpath, bootcsum_dfd, kernel_layout->kernel_namever, sysroot->debug_flags, cancellable, error)) @@ -1683,7 +1720,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, return FALSE; if (errno == ENOENT) { - if (!install_into_boot (sepolicy, kernel_layout->boot_dfd, kernel_layout->initramfs_srcpath, + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->initramfs_srcpath, bootcsum_dfd, kernel_layout->initramfs_namever, sysroot->debug_flags, cancellable, error)) @@ -1698,7 +1735,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, return FALSE; if (errno == ENOENT) { - if (!install_into_boot (sepolicy, kernel_layout->boot_dfd, kernel_layout->devicetree_srcpath, + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->devicetree_srcpath, bootcsum_dfd, kernel_layout->devicetree_namever, sysroot->debug_flags, cancellable, error)) @@ -1712,7 +1749,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, return FALSE; if (errno == ENOENT) { - if (!install_into_boot (sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_hmac_srcpath, + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->kernel_hmac_srcpath, bootcsum_dfd, kernel_layout->kernel_hmac_namever, sysroot->debug_flags, cancellable, error)) From 7febd9d36ecb52292ec93f6c7697470bb40d86ed Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 3 Feb 2020 15:42:28 +0000 Subject: [PATCH 38/49] Add .cci.jenkinsfile See https://github.com/jlebon/coreos-ci This is just a start to test, cut down from the rpm-ostree version. --- .cci.jenkinsfile | 36 +++++++++++++++++++++++++++++++ ci/ci-commitmessage-submodules.sh | 6 ++++-- 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 .cci.jenkinsfile diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile new file mode 100644 index 00000000..9acb34c5 --- /dev/null +++ b/.cci.jenkinsfile @@ -0,0 +1,36 @@ +// See https://github.com/jlebon/coreos-ci +// This was originally copied from the rpm-ostree one +@Library('github.com/coreos/coreos-ci-lib@master') _ + +COSA_IMAGE = 'quay.io/coreos-assembler/coreos-assembler:latest' +COSA_BUILDROOT_IMAGE = 'registry.svc.ci.openshift.org/coreos/cosa-buildroot:latest' + +stage("Build") { +parallel rpms: { + coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + checkout scm + sh """ + set -xeuo pipefail + # fetch tags so `git describe` gives a nice NEVRA when building the RPM + git fetch origin --tags + git submodule update --init + + env NOCONFIGURE=1 ./autogen.sh + ./configure --with-openssl --with-curl --with-selinux + make + """ + } +}, +codestyle: { + coreos.pod(image: COSA_IMAGE) { + checkout scm + sh """ + set -xeuo pipefail + # Jenkins by default only fetches the branch it's testing. Explicitly fetch master + # for ci-commitmessage-submodules.sh + git fetch origin +refs/heads/master:refs/remotes/origin/master + ci/ci-commitmessage-submodules.sh + """ + } +} +} diff --git a/ci/ci-commitmessage-submodules.sh b/ci/ci-commitmessage-submodules.sh index 38481aaf..35d828e4 100755 --- a/ci/ci-commitmessage-submodules.sh +++ b/ci/ci-commitmessage-submodules.sh @@ -29,8 +29,10 @@ cleanup_tmp() { } trap cleanup_tmp EXIT -pkg_upgrade -pkg_install git +if ! [ -x /usr/bin/git ]; then + pkg_upgrade + pkg_install git +fi gitdir=$(realpath $(pwd)) # Create a temporary copy of this (using cp not git clone) so git doesn't From 5a5c1e5b65151ae0c99d35d544cc3164be3da051 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 3 Feb 2020 21:03:17 +0000 Subject: [PATCH 39/49] travis: Update debian/ubuntu environments There are new major LTS environments out; bump up to the latest for each and drop the old Ubuntu trusty. Part of cleaning up our CI. --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2342b0f..a77bf827 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,10 @@ dist: trusty sudo: required env: - - ci_distro=ubuntu ci_suite=trusty ci_test=no # TODO: use libcurl on this - - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch - - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl" - - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_test=no # TODO gpgme flake https://github.com/ostreedev/ostree/pull/664#issuecomment-276033383 + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl" - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial + - ci_docker=ubuntu:bionic ci_distro=ubuntu ci_suite=bionic script: - ci/travis-install.sh From 6d104f6257a52c0ecbadd285187c920f86f2387c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 3 Feb 2020 20:45:07 +0000 Subject: [PATCH 40/49] ci: Replace PAPR with CoreOS CI Move the alternative builds into the Jenkinsfile. Update it to do a FCOS build + kola run. We drop the flatpak/rpm-ostree runs for now; the former will needs some work to do the automatic virt bits. The latter I think we can circle back to when we e.g. figure out how to include rpm-ostree's tests in kola runs. --- .cci.jenkinsfile | 86 ++++++++++++++++++- .papr-ex.yaml | 120 --------------------------- .papr.yml | 145 --------------------------------- tests/test-libarchive-import.c | 6 ++ 4 files changed, 90 insertions(+), 267 deletions(-) delete mode 100644 .papr-ex.yaml delete mode 100644 .papr.yml diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 9acb34c5..96e52441 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -6,17 +6,49 @@ COSA_IMAGE = 'quay.io/coreos-assembler/coreos-assembler:latest' COSA_BUILDROOT_IMAGE = 'registry.svc.ci.openshift.org/coreos/cosa-buildroot:latest' stage("Build") { -parallel rpms: { +parallel normal: { coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { checkout scm + stage("Core build") { sh """ set -xeuo pipefail # fetch tags so `git describe` gives a nice NEVRA when building the RPM git fetch origin --tags git submodule update --init + env SKIP_INSTALLDEPS=1 ./ci/build.sh + """ + } + stage("Unit tests") { + try { + sh """ + make check + make syntax-check + """ + } finally { + archiveArtifacts allowEmptyArchive: true, artifacts: 'test-suite.log' + } + } + sh """ + make install DESTDIR=\$(pwd)/insttree/ + tar -c -C insttree/ -zvf insttree.tar.gz . + """ + stash includes: 'insttree.tar.gz', name: 'build' + } +}, +// A minimal build, helps test our build options +minimal: { + coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + checkout scm + sh """ + set -xeuo pipefail + git submodule update --init + env NOCONFIGURE=1 ./autogen.sh - ./configure --with-openssl --with-curl --with-selinux + ./configure --without-curl --without-soup --disable-gtk-doc --disable-man \ + --disable-rust --without-libarchive --without-selinux --without-smack \ + --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse \ + --disable-experimental-api make """ } @@ -34,3 +66,53 @@ codestyle: { } } } + +// Build FCOS and do a kola basic run +stage("More builds and test") { +parallel fcos: { + coreos.pod(image: COSA_IMAGE, runAsUser: 0, kvm: true, memory: "2048Mi", cpu: "2") { + stage("Build FCOS") { + checkout scm + unstash 'build' + sh """ + set -xeuo pipefail + mkdir insttree + tar -C insttree -xzvf insttree.tar.gz + rsync -rlv insttree/ / + coreos-assembler init --force https://github.com/coreos/fedora-coreos-config + mkdir -p overrides/rootfs + mv insttree/* overrides/rootfs/ + rmdir insttree + coreos-assembler build + """ + } + stage("FCOS basic") { + timeout(time: 30, unit: 'MINUTES') { + sh """ + set -xeuo pipefail + cosa kola run --basic-qemu-scenarios + """ + } + } + } +}, +buildopts: { + coreos.pod(image: COSA_BUILDROOT_IMAGE, runAsUser: 0) { + checkout scm + sh """ + set -xeuo pipefail + git submodule update --init + + git worktree add build-rust && cd build-rust + env CONFIGOPTS="--enable-rust" SKIP_INSTALLDEPS=1 ./ci/build.sh + make check TESTS=tests/test-rollsum + cd .. && rm -rf build-rust + + git worktree add build-libsoup && cd build-libsoup + env CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh + make check + cd .. && rm -rf build-libsoup + """ + } +} +} diff --git a/.papr-ex.yaml b/.papr-ex.yaml deleted file mode 100644 index fbbd19dd..00000000 --- a/.papr-ex.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# https://fedoraproject.org/wiki/CI/Tests -branches: - - master - - auto - - try - -context: FAH29-insttests -required: false - -container: - image: registry.fedoraproject.org/fedora:29 - kvm: true - -tests: - - ci/fah29-insttests.sh - -artifacts: - - tests/installed/artifacts/ - ---- - -# This suite skips the RPMs and does the build+unit tests in a container -inherit: false -branches: - - master - - auto - - try -required: true -container: - image: registry.fedoraproject.org/fedora:29 -context: f29-primary -env: - # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain - CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' - # Only for CI with a known g-ir-scanner - GI_SCANNERFLAGS: '--warn-error' - ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc - # TODO when we're doing leak checks: G_SLICE: "always-malloc" - CONFIGOPTS: '--with-curl --with-openssl' - -tests: - - ci/ci-commitmessage-submodules.sh - - ci/build-check.sh - - ci/ci-release-build.sh - -artifacts: - - test-suite.log - - config.log - - gdtr-results - ---- -# And now the contexts below here are variant container builds - -context: f29-rust -inherit: true -container: - image: registry.fedoraproject.org/fedora:29 -env: - CONFIGOPTS: '--enable-rust' - CI_PKGS: cargo - -tests: - - ci/build.sh - - make check TESTS=tests/test-rollsum - ---- - -context: f29-gnutls -inherit: true -container: - image: registry.fedoraproject.org/fedora:29 -env: - CONFIGOPTS: '--with-crypto=gnutls' - CI_PKGS: pkgconfig(gnutls) - -tests: - - ci/build.sh - - make check TESTS=tests/test-basic.sh - ---- - -inherit: true - -context: f29-minimal -env: - CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man - --disable-rust --without-libarchive --without-selinux --without-smack - --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse - --disable-experimental-api' - -tests: - - ci/build.sh - ---- - -inherit: true -required: true - -context: f29-libsoup - -env: - CONFIGOPTS: "--without-curl --without-openssl --with-soup" - -tests: - - ci/build-check.sh - ---- - -inherit: true -required: true - -context: f29-introspection-tests - -env: - # ASAN conflicts with introspection testing; - # See https://github.com/ostreedev/ostree/issues/1014 - INSTALLED_TESTS_PATTERN: "libostree/test-sizes.js libostree/test-sysroot.js libostree/test-core.js" - -tests: - - ci/build-check.sh diff --git a/.papr.yml b/.papr.yml deleted file mode 100644 index f74985e4..00000000 --- a/.papr.yml +++ /dev/null @@ -1,145 +0,0 @@ -# This suite skips the RPMs and does the build+unit tests in a container -inherit: false - -container: - image: registry.fedoraproject.org/fedora:29 -context: f29-primary -env: - # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain - CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' - # Only for CI with a known g-ir-scanner - GI_SCANNERFLAGS: '--warn-error' - ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc - # TODO when we're doing leak checks: G_SLICE: "always-malloc" - CONFIGOPTS: '--with-curl --with-openssl' - -tests: - - ci/ci-commitmessage-submodules.sh - - ci/build-check.sh - - ci/ci-release-build.sh - - make dist-then-build - -artifacts: - - test-suite.log - - config.log - - gdtr-results - ---- -# And now the contexts below here are variant container builds - -context: f29-rust -inherit: true -container: - image: registry.fedoraproject.org/fedora:29 -env: - CONFIGOPTS: '--enable-rust' - CI_PKGS: cargo - -tests: - - ci/build.sh - - make check TESTS=tests/test-rollsum - ---- - -context: f29-gnutls -inherit: true -container: - image: registry.fedoraproject.org/fedora:29 -env: - CONFIGOPTS: '--with-crypto=gnutls' - CI_PKGS: pkgconfig(gnutls) - -tests: - - ci/build.sh - - make check TESTS=tests/test-basic.sh - ---- - -inherit: true - -context: f29-minimal -env: - CONFIGOPTS: '--without-curl --without-soup --disable-gtk-doc --disable-man - --disable-rust --without-libarchive --without-selinux --without-smack - --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse - --disable-experimental-api' - -tests: - - ci/build.sh - ---- - -inherit: true - -context: f29-libsoup - -env: - CONFIGOPTS: "--without-curl --without-openssl --with-soup" - -tests: - - ci/build-check.sh - ---- - -inherit: true - -context: f29-introspection-tests - -env: - # ASAN conflicts with introspection testing; - # See https://github.com/ostreedev/ostree/issues/1014 - INSTALLED_TESTS_PATTERN: "libostree/test-sizes.js libostree/test-sysroot.js libostree/test-core.js" - -tests: - - ci/build-check.sh - ---- - -# Reset inheritance for non-variant builds -inherit: false - -context: f29-flatpak - -# This test case wants an "unprivileged container with bubblewrap", -# which we don't have right now; so just provision a VM and do a -# docker --privileged run. -host: - distro: fedora/29/atomic - specs: - ram: 4096 # build-bundle is a static delta, which needs RAM right now - -tests: - - docker run --rm --privileged -v $(pwd):/srv/code registry.fedoraproject.org/fedora:29 /bin/sh -c "cd /srv/code && ./ci/flatpak.sh" - -artifacts: - - test-suite.log - ---- - -# Run rpm-ostree's vmcheck. This is a temporary hack until -# we share more code. https://github.com/projectatomic/rpm-ostree/issues/662 -inherit: false - -context: f29-rpmostree - -cluster: - hosts: - - name: vmcheck - distro: fedora/29/atomic - container: - image: registry.fedoraproject.org/fedora:29 - -env: - HOSTS: vmcheck - # This should roughly match the Fedora spec file, although right now we don't - # explicitly enable gtk-doc because we don't really need it - CONFIGOPTS: '--with-selinux --with-dracut=yesbutnoconf --with-curl --with-openssl' - -tests: - - ./ci/rpmostree.sh - -artifacts: - - test-suite.log - - vmcheck - -timeout: 60m diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c index 70e221d1..d3429cd6 100644 --- a/tests/test-libarchive-import.c +++ b/tests/test-libarchive-import.c @@ -544,6 +544,12 @@ test_libarchive_selinux (gconstpointer data) if (skip_if_no_xattr (td)) goto out; + if (getenv ("container")) + { + // FIXME dedup this with libtest.sh have_selinux_relabel + g_test_skip ("skip in containers for now"); + goto out; + } if (td->skip_all != NULL) { From 4b644bc671d340172d8d42c6472301d3c28f836d Mon Sep 17 00:00:00 2001 From: Cole Robinson Date: Mon, 10 Feb 2020 14:00:11 -0500 Subject: [PATCH 41/49] docs: Fix 'package layering' rpm-ostree link Signed-off-by: Cole Robinson --- docs/manual/adapting-existing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md index 14b36a88..3a1b8d69 100644 --- a/docs/manual/adapting-existing.md +++ b/docs/manual/adapting-existing.md @@ -166,7 +166,7 @@ Then to actually deploy this tree for the next boot: `ostree admin deploy $osname/$releasename/$description` This is essentially what [rpm-ostree](https://github.com/projectatomic/rpm-ostree/) -does to support its [package layering model](https://rpm-ostree.readthedocs.io/en/latest/manual/administrator-handbook/#package-layering). +does to support its [package layering model](https://rpm-ostree.readthedocs.io/en/latest/manual/administrator-handbook/#hybrid-imagepackaging-via-package-layering). ###### Licensing for this document: `SPDX-License-Identifier: (CC-BY-SA-3.0 OR GFDL-1.3-or-later)` From 0dd8dec2c996f4b16884f03c9de304dde9eac105 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Feb 2020 16:35:50 +0000 Subject: [PATCH 42/49] deploy: Avoid trying to change immutable state unnecessarily For some reason I haven't fully debugged (probably a recent kernel change), in the case where the immutable bit isn't set, trying to call `EXT2_IOC_SETFLAGS` without it set returns `EINVAL`. Let's avoid calling the `ioctl()` if we don't have anything to do. This fixes a slew of `make check` failures here in my toolbox environment. (kernel is `5.5.0-0.rc6.git0.1.fc32.x86_64` with `xfs`) --- src/libostree/ostree-linuxfsutil.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libostree/ostree-linuxfsutil.c b/src/libostree/ostree-linuxfsutil.c index d1dde8b4..231ecf76 100644 --- a/src/libostree/ostree-linuxfsutil.c +++ b/src/libostree/ostree-linuxfsutil.c @@ -68,6 +68,10 @@ _ostree_linuxfs_fd_alter_immutable_flag (int fd, } else { + gboolean prev_immutable_state = (flags & EXT2_IMMUTABLE_FL) > 0; + if (prev_immutable_state == new_immutable_state) + return TRUE; /* Nothing to do */ + if (new_immutable_state) flags |= EXT2_IMMUTABLE_FL; else From 7085a50297955b037b6433821796f3f317c79126 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 12 Dec 2019 16:40:33 -0500 Subject: [PATCH 43/49] lib/repo: Create repo directories as 0775 For repo structure directories like `objects`, `refs`, etc... we should be more permissive and let the system's `umask` narrow down the permission bits as wanted. This came up in a context where we want to be able to have read/write access on an OSTree repo on NFS from two separate OpenShift apps by using supplemental groups[1] so we don't require SCCs for running as the same UID (supplemental groups are part of the default restricted SCC). [1] https://docs.openshift.com/container-platform/3.11/install_config/persistent_storage/persistent_storage_nfs.html#nfs-supplemental-groups --- src/libostree/ostree-core-private.h | 6 ++++++ src/libostree/ostree-repo-checkout.c | 4 ++-- src/libostree/ostree-repo-pull.c | 2 +- src/libostree/ostree-repo-refs.c | 2 +- src/libostree/ostree-repo-static-delta-compilation.c | 2 +- src/libostree/ostree-repo.c | 12 ++++++------ 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 43cf22c4..7be97c8f 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -30,6 +30,12 @@ G_BEGIN_DECLS /* It's what gzip does, 9 is too slow */ #define OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL (6) +/* Note the permissive group bits. We want to be liberal here and let individual machines + * narrow permissions as needed via umask. This is important in setups where group ownership + * can matter for repo management (like OpenShift). */ +#define DEFAULT_DIRECTORY_MODE 0775 +#define DEFAULT_REGFILE_MODE 0660 + /* This file contains private implementation data format definitions * read by multiple implementation .c files. */ diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 8dd14640..dc36370f 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -92,8 +92,8 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, if (self->uncompressed_objects_dir_fd == -1) { - if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, "uncompressed-objects-cache", 0755, - cancellable, error)) + if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, "uncompressed-objects-cache", + DEFAULT_DIRECTORY_MODE, cancellable, error)) return FALSE; if (!glnx_opendirat (self->repo_dir_fd, "uncompressed-objects-cache", TRUE, &self->uncompressed_objects_dir_fd, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 586b34fe..125c113d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2925,7 +2925,7 @@ _ostree_repo_cache_summary (OstreeRepo *self, if (self->cache_dir_fd == -1) return TRUE; - if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, 0775, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, DEFAULT_DIRECTORY_MODE, cancellable, error)) return FALSE; const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote); diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 536a763a..d1d679b2 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -1181,7 +1181,7 @@ _ostree_repo_write_ref (OstreeRepo *self, char *parent = strdupa (ref->ref_name); parent[lastslash - ref->ref_name] = '\0'; - if (!glnx_shutil_mkdir_p_at (dfd, parent, 0755, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (dfd, parent, DEFAULT_DIRECTORY_MODE, cancellable, error)) return FALSE; } diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 054ac06f..88e9ddf6 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1427,7 +1427,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autofree char *dnbuf = g_strdup (descriptor_relpath); const char *dn = dirname (dnbuf); - if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, dn, 0755, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, dn, DEFAULT_DIRECTORY_MODE, cancellable, error)) goto out; if (!glnx_opendirat (self->repo_dir_fd, dn, TRUE, &descriptor_dfd, error)) goto out; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index cff70d47..304e9f48 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -357,7 +357,7 @@ push_repo_lock (OstreeRepo *self, g_debug ("Opening repo lock file"); lock->fd = TEMP_FAILURE_RETRY (openat (self->repo_dir_fd, ".lock", O_CREAT | O_RDWR | O_CLOEXEC, - 0600)); + DEFAULT_REGFILE_MODE)); if (lock->fd < 0) { free_repo_lock (lock); @@ -2489,7 +2489,7 @@ repo_create_at_internal (int dfd, } } - if (mkdirat (dfd, path, 0755) != 0) + if (mkdirat (dfd, path, DEFAULT_DIRECTORY_MODE) != 0) { if (G_UNLIKELY (errno != EEXIST)) return glnx_throw_errno_prefix (error, "mkdirat"); @@ -2527,7 +2527,7 @@ repo_create_at_internal (int dfd, for (guint i = 0; i < G_N_ELEMENTS (state_dirs); i++) { const char *elt = state_dirs[i]; - if (mkdirat (repo_dfd, elt, 0755) == -1) + if (mkdirat (repo_dfd, elt, DEFAULT_DIRECTORY_MODE) == -1) { if (G_UNLIKELY (errno != EEXIST)) return glnx_throw_errno_prefix (error, "mkdirat"); @@ -3295,7 +3295,7 @@ ostree_repo_open (OstreeRepo *self, * * https://github.com/ostreedev/ostree/issues/1018 */ - if (mkdirat (self->repo_dir_fd, "tmp", 0755) == -1) + if (mkdirat (self->repo_dir_fd, "tmp", DEFAULT_DIRECTORY_MODE) == -1) { if (G_UNLIKELY (errno != EEXIST)) return glnx_throw_errno_prefix (error, "mkdir(tmp)"); @@ -3307,7 +3307,7 @@ ostree_repo_open (OstreeRepo *self, if (self->writable) { - if (!glnx_shutil_mkdir_p_at (self->tmp_dir_fd, _OSTREE_CACHE_DIR, 0775, cancellable, error)) + if (!glnx_shutil_mkdir_p_at (self->tmp_dir_fd, _OSTREE_CACHE_DIR, DEFAULT_DIRECTORY_MODE, cancellable, error)) return FALSE; if (!glnx_opendirat (self->tmp_dir_fd, _OSTREE_CACHE_DIR, TRUE, &self->cache_dir_fd, error)) @@ -6099,7 +6099,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, { g_auto(GLnxTmpDir) new_tmpdir = { 0, }; /* No existing tmpdir found, create a new */ - if (!glnx_mkdtempat (tmpdir_dfd, tmpdir_name_template, 0755, + if (!glnx_mkdtempat (tmpdir_dfd, tmpdir_name_template, DEFAULT_DIRECTORY_MODE, &new_tmpdir, error)) return FALSE; From 8e65366ffd96f53d4d5acc5be009d584026ae783 Mon Sep 17 00:00:00 2001 From: clime Date: Tue, 18 Feb 2020 21:46:22 +0100 Subject: [PATCH 44/49] Update ostree-pull.xml with info about pulled refs location and access --- man/ostree-pull.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index 84d75be3..0606f690 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -143,6 +143,14 @@ Boston, MA 02111-1307, USA. Description + + Without --mirror, this command will create new refs + under remotes/REMOTE/ directory + for each pulled branch unless they are already created. Such + refs can be then referenced by REMOTE:BRANCH in + ostree subcommands (e.g. ostree log origin:exampleos/x86_64/standard). + + This command can retrieve just a specific commit, or go all the way to performing a full mirror of the remote From d5bfbc6715e8bc199a33bfb94c143261c4c53364 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Feb 2020 00:33:01 +0000 Subject: [PATCH 45/49] prepare-root: Add a comment about the role of this service Came up on an IRC question, docs for this are scattered around. --- src/switchroot/ostree-prepare-root.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 33c46ff4..c25d3fe9 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -29,6 +29,28 @@ * Boston, MA 02111-1307, USA. */ +/* The high level goal of ostree-prepare-root.service is to run inside + * the initial ram disk (if one is in use) and set up the `/` mountpoint + * to be the deployment root, using the ostree= kernel commandline + * argument to find the target deployment root. + * + * It's really the heart of how ostree works - basically multiple + * hardlinked chroot() targets are maintained, this one does the equivalent + * of chroot(). + * + * If using systemd, an excellent reference is `man bootup`. This + * service runs Before=initrd-root-fs.target. At this point it's + * assumed that the block storage and root filesystem are mounted at + * /sysroot - i.e. /sysroot points to the *physical* root before + * this service runs. After, `/` is the deployment root. + * + * There is also a secondary mode for this service when an initrd isn't + * used - instead the binary must be statically linked (and the kernel + * must have mounted the rootfs itself) - then we set things up and + * exec the real init directly. This can be popular in embedded + * systems to increase bootup speed. + */ + #include "config.h" #include From 04c85fa101996472158f5361d2a7632d77ff6ec2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Feb 2020 16:16:19 +0000 Subject: [PATCH 46/49] Release 2020.1 New year, new release! --- configure.ac | 6 +++--- src/libostree/libostree-devel.sym | 17 +++++------------ src/libostree/libostree-released.sym | 12 ++++++++++++ tests/test-symbols.sh | 2 +- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/configure.ac b/configure.ac index a28172a1..4824e7c0 100644 --- a/configure.ac +++ b/configure.ac @@ -2,12 +2,12 @@ AC_PREREQ([2.63]) dnl To do a release: follow the instructions to update libostree-released.sym from dnl libostree-devel.sym, update the checksum in test-symbols.sh, set is_release_build=yes dnl below. Then make another post-release commit to bump the version and set -dnl is_release_build=no. +dnl is_release_build=yes dnl Seed the release notes with `git-shortlog-with-prs ..`. Then use dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. -m4_define([year_version], [2019]) -m4_define([release_version], [7]) +m4_define([year_version], [2020]) +m4_define([release_version], [1]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) is_release_build=no diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index ff5f52c4..aa3392cc 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,24 +18,17 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2019.7 { +LIBOSTREE_2020.2 { global: - ostree_commit_get_object_sizes; - ostree_commit_sizes_entry_copy; - ostree_commit_sizes_entry_free; - ostree_commit_sizes_entry_get_type; - ostree_commit_sizes_entry_new; - ostree_sysroot_initialize; - ostree_sysroot_is_booted; - ostree_sysroot_set_mount_namespace_in_use; -} LIBOSTREE_2019.6; + someostree_symbol_deleteme; +} LIBOSTREE_2020.1; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION * with whatever the next version with new symbols will be. -LIBOSTREE_2019.$NEWVERSION { +LIBOSTREE_2020.$NEWVERSION { global: someostree_symbol_deleteme; -} LIBOSTREE_2019.$LASTSTABLE; +} LIBOSTREE_2020.$LASTSTABLE; */ diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index e81a6595..8fda31ef 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -581,6 +581,18 @@ LIBOSTREE_2019.6 { ostree_async_progress_copy_state; } LIBOSTREE_2019.4; +LIBOSTREE_2020.1 { +global: + ostree_commit_get_object_sizes; + ostree_commit_sizes_entry_copy; + ostree_commit_sizes_entry_free; + ostree_commit_sizes_entry_get_type; + ostree_commit_sizes_entry_new; + ostree_sysroot_initialize; + ostree_sysroot_is_booted; + ostree_sysroot_set_mount_namespace_in_use; +} LIBOSTREE_2019.6; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index a041ddb1..cbcb6c69 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -54,7 +54,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt < Date: Thu, 20 Feb 2020 16:37:07 +0000 Subject: [PATCH 47/49] Post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 4824e7c0..d175fa03 100644 --- a/configure.ac +++ b/configure.ac @@ -2,12 +2,12 @@ AC_PREREQ([2.63]) dnl To do a release: follow the instructions to update libostree-released.sym from dnl libostree-devel.sym, update the checksum in test-symbols.sh, set is_release_build=yes dnl below. Then make another post-release commit to bump the version and set -dnl is_release_build=yes +dnl is_release_build=no dnl Seed the release notes with `git-shortlog-with-prs ..`. Then use dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2020]) -m4_define([release_version], [1]) +m4_define([release_version], [2]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) is_release_build=no From 3557e4e1652fdae19a18099e08dfe65ef1cf9cd7 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 20 Feb 2020 15:44:51 -0700 Subject: [PATCH 48/49] lib: Fix Since versions for 2020.1 These had been added assuming 2019.7 would be the next version, but now it's 2020 and there's been a release. In the case of `OstreeCommitSizesEntry`, I'd forgotten to move it forward from 2019.5 to 2019.7 in the time between when I started working on the feature and it landed. --- src/libostree/ostree-core.c | 8 ++++---- src/libostree/ostree-core.h | 2 +- src/libostree/ostree-gpg-verify-result.h | 6 +++--- src/libostree/ostree-sysroot.c | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 4667dd8f..48815bd3 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -2445,7 +2445,7 @@ G_DEFINE_BOXED_TYPE (OstreeCommitSizesEntry, ostree_commit_sizes_entry, * commit's "ostree.sizes" metadata. * * Returns: (transfer full) (nullable): a new #OstreeCommitSizesEntry - * Since: 2019.7 + * Since: 2020.1 */ OstreeCommitSizesEntry * ostree_commit_sizes_entry_new (const gchar *checksum, @@ -2471,7 +2471,7 @@ ostree_commit_sizes_entry_new (const gchar *checksum, * Create a copy of the given @entry. * * Returns: (transfer full) (nullable): a new copy of @entry - * Since: 2019.7 + * Since: 2020.1 */ OstreeCommitSizesEntry * ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry) @@ -2490,7 +2490,7 @@ ostree_commit_sizes_entry_copy (const OstreeCommitSizesEntry *entry) * * Free given @entry. * - * Since: 2019.7 + * Since: 2020.1 */ void ostree_commit_sizes_entry_free (OstreeCommitSizesEntry *entry) @@ -2572,7 +2572,7 @@ read_sizes_entry (GVariant *entry, * the "ostree.sizes" metadata, a %G_IO_ERROR_NOT_FOUND error will be * returned. * - * Since: 2019.7 + * Since: 2020.1 */ gboolean ostree_commit_get_object_sizes (GVariant *commit_variant, diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 10601123..540f3232 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -531,7 +531,7 @@ gchar * ostree_commit_get_content_checksum (GVariant *commit_variant); * Structure representing an entry in the "ostree.sizes" commit metadata. Each * entry corresponds to an object in the associated commit. * - * Since: 2019.5 + * Since: 2020.1 */ typedef struct { gchar *checksum; diff --git a/src/libostree/ostree-gpg-verify-result.h b/src/libostree/ostree-gpg-verify-result.h index f71ab981..6f51ce8a 100644 --- a/src/libostree/ostree-gpg-verify-result.h +++ b/src/libostree/ostree-gpg-verify-result.h @@ -159,11 +159,11 @@ gboolean ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult * @OSTREE_GPG_ERROR_NO_SIGNATURE: A signature was expected, but not found. * @OSTREE_GPG_ERROR_INVALID_SIGNATURE: A signature was malformed. * @OSTREE_GPG_ERROR_MISSING_KEY: A signature was found, but was created with a key not in the configured keyrings. - * @OSTREE_GPG_ERROR_EXPIRED_SIGNATURE: A signature was expired. Since: 2019.7. + * @OSTREE_GPG_ERROR_EXPIRED_SIGNATURE: A signature was expired. Since: 2020.1. * @OSTREE_GPG_ERROR_EXPIRED_KEY: A signature was found, but the key used to - * sign it has expired. Since: 2019.7. + * sign it has expired. Since: 2020.1. * @OSTREE_GPG_ERROR_REVOKED_KEY: A signature was found, but the key used to - * sign it has been revoked. Since: 2019.7. + * sign it has been revoked. Since: 2020.1. * * Errors returned by signature creation and verification operations in OSTree. * These may be returned by any API which creates or verifies signatures. diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 23a06975..55a61014 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -242,7 +242,7 @@ ostree_sysroot_new_default (void) * If you invoke this function, it must be before ostree_sysroot_load(); it may * be invoked before or after ostree_sysroot_initialize(). * - * Since: 2019.7 + * Since: 2020.1 */ void ostree_sysroot_set_mount_namespace_in_use (OstreeSysroot *self) @@ -341,7 +341,7 @@ ostree_sysroot_get_fd (OstreeSysroot *self) * Can only be invoked after `ostree_sysroot_initialize()`. * * Returns: %TRUE iff the sysroot points to a booted deployment - * Since: 2019.7 + * Since: 2020.1 */ gboolean ostree_sysroot_is_booted (OstreeSysroot *self) @@ -890,7 +890,7 @@ ensure_repo (OstreeSysroot *self, * It is not necessary to call this function if ostree_sysroot_load() is * invoked. * - * Since: 2019.7 + * Since: 2020.1 */ gboolean ostree_sysroot_initialize (OstreeSysroot *self, From c6085ebd5e27da35f43165eb614190665468f13a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Feb 2020 14:39:14 +0000 Subject: [PATCH 49/49] Release 2020.2 "Brown paper bag" release that actually sets the `is_release_build=yes` flag and also fixes the `Since:` on a few new functions. --- configure.ac | 4 ++-- src/libostree/libostree-released.sym | 2 ++ tests/test-symbols.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index d175fa03..36995403 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ([2.63]) dnl To do a release: follow the instructions to update libostree-released.sym from dnl libostree-devel.sym, update the checksum in test-symbols.sh, set is_release_build=yes dnl below. Then make another post-release commit to bump the version and set -dnl is_release_build=no +dnl is_release_build=yes dnl Seed the release notes with `git-shortlog-with-prs ..`. Then use dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. @@ -10,7 +10,7 @@ m4_define([year_version], [2020]) m4_define([release_version], [2]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index 8fda31ef..33d4d0e3 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -593,6 +593,8 @@ global: ostree_sysroot_set_mount_namespace_in_use; } LIBOSTREE_2019.6; +/* No new symbols in 2020.2 */ + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index cbcb6c69..f6742d93 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -54,7 +54,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <