From 1f3f6577e230ed2cb0e77b4abaa8d7dd1ae01236 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 16:56:08 -0400 Subject: [PATCH 01/80] Post-release version bump Closes: #1512 Approved by: jlebon --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 35962dfa..6c7a8b5c 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2018]) -m4_define([release_version], [4]) +m4_define([release_version], [5]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 0b1d3b87e363e548546f2a39e7edb59f783e429c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 14:57:33 -0400 Subject: [PATCH 02/80] lib/deltas: Some misc declare-and-initialize porting Also some `glnx_fstatat_allow_noent()`. Not specifically prep for anything. Closes: #1511 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-core.c | 130 +++++++----------- 1 file changed, 50 insertions(+), 80 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 68b06b5c..57c89736 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -104,11 +104,6 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, while (TRUE) { struct dirent *sub_dent; - const char *name1; - const char *name2; - g_autofree char *superblock_subpath = NULL; - struct stat stbuf; - if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent, cancellable, error)) return FALSE; @@ -117,39 +112,33 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, if (dent->d_type != DT_DIR) continue; - name1 = dent->d_name; - name2 = sub_dent->d_name; + const char *name1 = dent->d_name; + const char *name2 = sub_dent->d_name; - superblock_subpath = g_strconcat (name2, "/superblock", NULL); - if (fstatat (sub_dfd_iter.fd, superblock_subpath, &stbuf, 0) < 0) - { - if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - return FALSE; - } - } - else - { - g_autofree char *buf = g_strconcat (name1, name2, NULL); - GString *out = g_string_new (""); - char checksum[OSTREE_SHA256_STRING_LEN+1]; - guchar csum[OSTREE_SHA256_DIGEST_LEN]; - const char *dash = strchr (buf, '-'); + g_autofree char *superblock_subpath = g_strconcat (name2, "/superblock", NULL); + if (!glnx_fstatat_allow_noent (sub_dfd_iter.fd, superblock_subpath, NULL, 0, error)) + return FALSE; + if (errno == ENOENT) + continue; - ostree_checksum_b64_inplace_to_bytes (buf, csum); + g_autofree char *buf = g_strconcat (name1, name2, NULL); + GString *out = g_string_new (""); + char checksum[OSTREE_SHA256_STRING_LEN+1]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; + const char *dash = strchr (buf, '-'); + + ostree_checksum_b64_inplace_to_bytes (buf, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + g_string_append (out, checksum); + if (dash) + { + g_string_append_c (out, '-'); + ostree_checksum_b64_inplace_to_bytes (dash+1, csum); ostree_checksum_inplace_from_bytes (csum, checksum); g_string_append (out, checksum); - if (dash) - { - g_string_append_c (out, '-'); - ostree_checksum_b64_inplace_to_bytes (dash+1, csum); - ostree_checksum_inplace_from_bytes (csum, checksum); - g_string_append (out, checksum); - } - - g_ptr_array_add (ret_deltas, g_string_free (out, FALSE)); } + + g_ptr_array_add (ret_deltas, g_string_free (out, FALSE)); } } @@ -320,13 +309,10 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, guint32 version; guint64 size; guint64 usize; - const guchar *csum; char checksum[OSTREE_SHA256_STRING_LEN+1]; - gboolean have_all; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; g_autoptr(GVariant) part = NULL; - g_autofree char *deltapart_path = NULL; OstreeStaticDeltaOpenFlags delta_open_flags = skip_validation ? OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM : 0; g_autoptr(GVariant) header = g_variant_get_child_value (headers, i); @@ -335,6 +321,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, if (version > OSTREE_DELTAPART_VERSION) return glnx_throw (error, "Delta part has too new version %u", version); + gboolean have_all; if (!_ostree_repo_static_delta_part_have_all_objects (self, objects, &have_all, cancellable, error)) return FALSE; @@ -345,12 +332,12 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, if (have_all) continue; - csum = ostree_checksum_bytes_peek_validate (csum_v, error); + const guchar *csum = ostree_checksum_bytes_peek_validate (csum_v, error); if (!csum) return FALSE; ostree_checksum_inplace_from_bytes (csum, checksum); - deltapart_path = + g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i); g_autoptr(GInputStream) part_in = NULL; @@ -410,16 +397,14 @@ _ostree_static_delta_part_open (GInputStream *part_in, { const gboolean trusted = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED) > 0; const gboolean skip_checksum = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM) > 0; - gsize bytes_read; - guint8 comptype; - g_autoptr(GChecksum) checksum = NULL; - g_autoptr(GInputStream) checksum_in = NULL; - GInputStream *source_in; /* We either take a fd or a GBytes reference */ g_return_val_if_fail (G_IS_FILE_DESCRIPTOR_BASED (part_in) || inline_part_bytes != NULL, FALSE); g_return_val_if_fail (skip_checksum || expected_checksum != NULL, FALSE); + g_autoptr(GChecksum) checksum = NULL; + g_autoptr(GInputStream) checksum_in = NULL; + GInputStream *source_in; if (!skip_checksum) { checksum = g_checksum_new (G_CHECKSUM_SHA256); @@ -431,7 +416,9 @@ _ostree_static_delta_part_open (GInputStream *part_in, source_in = part_in; } + guint8 comptype; { guint8 buf[1]; + gsize bytes_read; /* First byte is compression type */ if (!g_input_stream_read_all (source_in, buf, sizeof(buf), &bytes_read, cancellable, error)) @@ -511,7 +498,6 @@ show_one_part (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GVariant) part = NULL; g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i); guint32 version; @@ -530,6 +516,7 @@ show_one_part (OstreeRepo *self, return glnx_throw_errno_prefix (error, "openat(%s)", part_path); g_autoptr(GInputStream) part_in = g_unix_input_stream_new (part_fd, FALSE); + g_autoptr(GVariant) part = NULL; if (!_ostree_static_delta_part_open (part_in, NULL, OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, NULL, @@ -576,19 +563,13 @@ OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock, gboolean *out_was_heuristic) { - guint8 endianness_char; - g_autoptr(GVariant) delta_meta = NULL; - g_autoptr(GVariantDict) delta_metadict = NULL; - guint64 total_size = 0; - guint64 total_usize = 0; - guint total_objects = 0; - - delta_meta = g_variant_get_child_value (superblock, 0); - delta_metadict = g_variant_dict_new (delta_meta); + g_autoptr(GVariant) delta_meta = g_variant_get_child_value (superblock, 0); + g_autoptr(GVariantDict) delta_metadict = g_variant_dict_new (delta_meta); if (out_was_heuristic) *out_was_heuristic = FALSE; + guint8 endianness_char; if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) { switch (endianness_char) @@ -605,15 +586,16 @@ _ostree_delta_get_endianness (GVariant *superblock, if (out_was_heuristic) *out_was_heuristic = TRUE; + guint64 total_size = 0; + guint64 total_usize = 0; + guint total_objects = 0; { g_autoptr(GVariant) meta_entries = NULL; - guint n_parts; - guint i; gboolean is_byteswapped = FALSE; g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); - n_parts = g_variant_n_children (meta_entries); + const guint n_parts = g_variant_n_children (meta_entries); - for (i = 0; i < n_parts; i++) + for (guint i = 0; i < n_parts; i++) { g_autoptr(GVariant) objects = NULL; guint64 size, usize; @@ -631,7 +613,7 @@ _ostree_delta_get_endianness (GVariant *superblock, double ratio = ((double)size)/((double)usize); /* This should really never happen where compressing things makes it more than 50% bigger. - */ + */ if (ratio > 1.2) { is_byteswapped = TRUE; @@ -724,26 +706,16 @@ _ostree_repo_static_delta_query_exists (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autofree char *from = NULL; + g_autofree char *from = NULL; g_autofree char *to = NULL; - struct stat stbuf; - if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) return FALSE; g_autofree char *superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); + if (!glnx_fstatat_allow_noent (self->repo_dir_fd, superblock_path, NULL, 0, error)) + return FALSE; - if (fstatat (self->repo_dir_fd, superblock_path, &stbuf, 0) < 0) - { - if (errno == ENOENT) - { - *out_exists = FALSE; - return TRUE; - } - else - return glnx_throw_errno_prefix (error, "fstatat(%s)", superblock_path); - } - *out_exists = TRUE; + *out_exists = (errno == 0); return TRUE; } @@ -755,21 +727,15 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, { g_autofree char *from = NULL; g_autofree char *to = NULL; - g_autofree char *superblock_path = NULL; - g_autoptr(GVariant) delta_superblock = NULL; - guint64 total_size = 0, total_usize = 0; - guint64 total_fallback_size = 0, total_fallback_usize = 0; - OstreeDeltaEndianness endianness; - gboolean swap_endian = FALSE; - if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) return FALSE; - superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); + g_autofree char *superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); glnx_autofd int superblock_fd = -1; if (!glnx_openat_rdonly (self->repo_dir_fd, superblock_path, TRUE, &superblock_fd, error)) return FALSE; + g_autoptr(GVariant) delta_superblock = NULL; if (!ot_variant_read_fd (superblock_fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, TRUE, &delta_superblock, error)) @@ -777,6 +743,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_print ("Delta: %s\n", delta_id); + gboolean swap_endian = FALSE; + OstreeDeltaEndianness endianness; { const char *endianness_description; gboolean was_heuristic; @@ -823,6 +791,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, g_print ("Number of fallback entries: %u\n", n_fallback); + guint64 total_size = 0, total_usize = 0; + guint64 total_fallback_size = 0, total_fallback_usize = 0; for (guint i = 0; i < n_fallback; i++) { guint64 size, usize; From 460fb7aebc23de4e11e4b9855bbbc866cb0f9430 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Mar 2018 15:25:14 -0400 Subject: [PATCH 03/80] lib/deploy: Set kargs in one place Prep for deployment staging. We had the code to hande "explicit kargs" in one place, but the "use merge deployment" karg bits mixed in with the "/etc merge" logic. Those are separate things, and it's better to have karg handling in one place. Closes: #1514 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 54 ++++++++++++--------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 93a29ed6..f6b562d2 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -759,20 +759,6 @@ merge_configuration (OstreeSysroot *sysroot, GLNX_AUTO_PREFIX_ERROR ("During /etc merge", error); g_autoptr(OstreeSePolicy) sepolicy = NULL; - if (previous_deployment) - { - OstreeBootconfigParser *previous_bootconfig = ostree_deployment_get_bootconfig (previous_deployment); - if (previous_bootconfig) - { - const char *previous_options = ostree_bootconfig_parser_get (previous_bootconfig, "options"); - /* Completely overwrite the previous options here; we will extend - * them later. - */ - ostree_bootconfig_parser_set (ostree_deployment_get_bootconfig (deployment), "options", - previous_options); - } - } - struct stat stbuf; if (!glnx_fstatat_allow_noent (deployment_dfd, "etc", &stbuf, AT_SYMLINK_NOFOLLOW, error)) return FALSE; @@ -2431,12 +2417,32 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, _ostree_deployment_set_bootcsum (new_deployment, kernel_layout->bootcsum); - /* Create an empty boot configuration; we will merge things into - * it as we go. - */ + /* Initial empty boot configuration. */ g_autoptr(OstreeBootconfigParser) bootconfig = ostree_bootconfig_parser_new (); ostree_deployment_set_bootconfig (new_deployment, bootconfig); + /* Handle kernel arguments. After this, install_deployment_kernel() will set + * the other boot options and write it out to disk. + */ + if (override_kernel_argv) + { + /* We have an override set, use it */ + g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_new (); + _ostree_kernel_args_append_argv (kargs, override_kernel_argv); + g_autofree char *new_options = _ostree_kernel_args_to_string (kargs); + ostree_bootconfig_parser_set (bootconfig, "options", new_options); + } + else if (provided_merge_deployment) + { + /* Use the merge options by default */ + OstreeBootconfigParser *merge_bootconfig = ostree_deployment_get_bootconfig (provided_merge_deployment); + if (merge_bootconfig) + { + const char *opts = ostree_bootconfig_parser_get (merge_bootconfig, "options"); + ostree_bootconfig_parser_set (bootconfig, "options", opts); + } + } + g_autoptr(OstreeSePolicy) sepolicy = NULL; if (!merge_configuration (self, repo, merge_deployment, new_deployment, deployment_dfd, @@ -2463,20 +2469,6 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, cancellable, error)) return FALSE; - /* After this, install_deployment_kernel() will set the other boot - * options and write it out to disk. - */ - if (override_kernel_argv) - { - g_autoptr(OstreeKernelArgs) kargs = NULL; - g_autofree char *new_options = NULL; - - kargs = _ostree_kernel_args_new (); - _ostree_kernel_args_append_argv (kargs, override_kernel_argv); - new_options = _ostree_kernel_args_to_string (kargs); - ostree_bootconfig_parser_set (bootconfig, "options", new_options); - } - ot_transfer_out_value (out_new_deployment, &new_deployment); return TRUE; } From 7ec3d0601430ebcc37006de2dc7a169fcc820646 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Mar 2018 15:33:38 -0400 Subject: [PATCH 04/80] lib/deploy: Split /etc merge into two stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For staged deploy, we want to pay the cost of creating copies from `/usr/etc` → `/etc` at stage time, since it can be expensive. (We want to minimize time spent during shutdown). Split it up into two functions; the logic is also simply clearer. Closes: #1514 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 63 +++++++++++---------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index f6b562d2..c94498de 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -390,7 +390,6 @@ copy_modified_config_file (int orig_etc_fd, * merge_configuration_from: * @sysroot: Sysroot * @merge_deployment: Source of configuration differences - * @merge_deployment_dfd: Directory fd, may be -1 * @new_deployment: Target for merge of configuration * @new_deployment_dfd: Directory fd for @new_deployment (may *not* be -1) * @cancellable: Cancellable @@ -407,27 +406,22 @@ copy_modified_config_file (int orig_etc_fd, static gboolean merge_configuration_from (OstreeSysroot *sysroot, OstreeDeployment *merge_deployment, - int merge_deployment_dfd, OstreeDeployment *new_deployment, int new_deployment_dfd, GCancellable *cancellable, GError **error) { - glnx_autofd int owned_merge_deployment_dfd = -1; + GLNX_AUTO_PREFIX_ERROR ("During /etc merge", error); const OstreeSysrootDebugFlags flags = sysroot->debug_flags; g_assert (merge_deployment != NULL && new_deployment != NULL); g_assert (new_deployment_dfd != -1); - /* Allow the caller to pass -1 for the merge, for convenience */ - if (merge_deployment_dfd == -1) - { - g_autofree char *merge_deployment_path = ostree_sysroot_get_deployment_dirpath (sysroot, merge_deployment); - if (!glnx_opendirat (sysroot->sysroot_fd, merge_deployment_path, FALSE, - &owned_merge_deployment_dfd, error)) - return FALSE; - merge_deployment_dfd = owned_merge_deployment_dfd; - } + g_autofree char *merge_deployment_path = ostree_sysroot_get_deployment_dirpath (sysroot, merge_deployment); + glnx_autofd int merge_deployment_dfd = -1; + if (!glnx_opendirat (sysroot->sysroot_fd, merge_deployment_path, FALSE, + &merge_deployment_dfd, error)) + return FALSE; /* TODO: get rid of GFile usage here */ g_autoptr(GFile) orig_etc = ot_fdrel_to_gfile (merge_deployment_dfd, "usr/etc"); @@ -741,22 +735,19 @@ selinux_relabel_var_if_needed (OstreeSysroot *sysroot, return TRUE; } -/* OSTree implements a "3 way" merge model for /etc. For a bit more information - * on this, see the manual. This function uses the configuration for - * @previous_deployment, and writes the merged configuration into @deployment's - * /etc. If available, we also load the SELinux policy from the new root. +/* Handle initial creation of /etc in the deployment. See also + * merge_configuration_from(). */ static gboolean -merge_configuration (OstreeSysroot *sysroot, - OstreeRepo *repo, - OstreeDeployment *previous_deployment, - OstreeDeployment *deployment, - int deployment_dfd, - OstreeSePolicy **out_sepolicy, - GCancellable *cancellable, - GError **error) +prepare_deployment_etc (OstreeSysroot *sysroot, + OstreeRepo *repo, + OstreeDeployment *deployment, + int deployment_dfd, + OstreeSePolicy **out_sepolicy, + GCancellable *cancellable, + GError **error) { - GLNX_AUTO_PREFIX_ERROR ("During /etc merge", error); + GLNX_AUTO_PREFIX_ERROR ("Preparing /etc", error); g_autoptr(OstreeSePolicy) sepolicy = NULL; struct stat stbuf; @@ -805,14 +796,6 @@ merge_configuration (OstreeSysroot *sysroot, } - if (previous_deployment) - { - if (!merge_configuration_from (sysroot, previous_deployment, -1, - deployment, deployment_dfd, - cancellable, error)) - return FALSE; - } - if (out_sepolicy) *out_sepolicy = g_steal_pointer (&sepolicy); return TRUE; @@ -2444,12 +2427,18 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, } g_autoptr(OstreeSePolicy) sepolicy = NULL; - if (!merge_configuration (self, repo, merge_deployment, new_deployment, - deployment_dfd, - &sepolicy, - cancellable, error)) + if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, + &sepolicy, cancellable, error)) return FALSE; + if (merge_deployment) + { + if (!merge_configuration_from (self, merge_deployment, + new_deployment, deployment_dfd, + cancellable, error)) + return FALSE; + } + if (!selinux_relabel_var_if_needed (self, sepolicy, os_deploy_dfd, cancellable, error)) return FALSE; From 2648c968c4dafaf758b9932234e19f0a3c1719d6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Mar 2018 16:02:38 -0400 Subject: [PATCH 05/80] lib/deploy: Port final bootconfig writing to new style The main blocker for doing this before was the `goto out` handling for remounting `/boot`. Handle that by factoring out the bits that require it to a helper function, and do the C/GError equivalent of "try/finally". Not prep for anything right now, just decided to do this since I had the file open. Closes: #1515 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 251 ++++++++++++-------------- 1 file changed, 120 insertions(+), 131 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index c94498de..927809e9 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1428,6 +1428,7 @@ full_system_sync (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Full sync", error); guint64 start_msec = g_get_monotonic_time () / 1000; if (syncfs (self->sysroot_fd) != 0) return glnx_throw_errno_prefix (error, "syncfs(sysroot)"); @@ -1819,6 +1820,7 @@ prepare_new_bootloader_link (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Preparing final bootloader swap", error); g_assert ((current_bootversion == 0 && new_bootversion == 1) || (current_bootversion == 1 && new_bootversion == 0)); @@ -1841,11 +1843,12 @@ swap_bootloader (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - glnx_autofd int boot_dfd = -1; + GLNX_AUTO_PREFIX_ERROR ("Final bootloader swap", error); g_assert ((current_bootversion == 0 && new_bootversion == 1) || (current_bootversion == 1 && new_bootversion == 0)); + glnx_autofd int boot_dfd = -1; if (!glnx_opendirat (sysroot->sysroot_fd, "boot", TRUE, &boot_dfd, error)) return FALSE; @@ -2032,6 +2035,91 @@ ostree_sysroot_write_deployments (OstreeSysroot *self, cancellable, error); } +/* Handle writing out a new bootloader config. One reason this needs to be a + * helper function is to handle wrapping it with temporarily remounting /boot + * rw. + */ +static gboolean +write_deployments_bootswap (OstreeSysroot *self, + GPtrArray *new_deployments, + OstreeSysrootWriteDeploymentsOpts *opts, + OstreeBootloader *bootloader, + SyncStats *out_syncstats, + GCancellable *cancellable, + GError **error) +{ + const int new_bootversion = self->bootversion ? 0 : 1; + + g_autofree char* new_loader_entries_dir = g_strdup_printf ("boot/loader.%d/entries", new_bootversion); + if (!glnx_shutil_rm_rf_at (self->sysroot_fd, new_loader_entries_dir, cancellable, error)) + return FALSE; + if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, new_loader_entries_dir, 0755, + cancellable, error)) + return FALSE; + + /* Need the repo to try and extract the versions for deployments. + * But this is a "nice-to-have" for the bootloader UI, so failure + * here is not fatal to the whole operation. We just gracefully + * fall back to the deployment index. */ + g_autoptr(OstreeRepo) repo = NULL; + (void) ostree_sysroot_get_repo (self, &repo, cancellable, NULL); + + /* Only show the osname in bootloader titles if there are multiple + * osname's among the new deployments. Check for that here. */ + gboolean show_osname = FALSE; + for (guint i = 1; i < new_deployments->len; i++) + { + const char *osname_0 = ostree_deployment_get_osname (new_deployments->pdata[0]); + const char *osname_i = ostree_deployment_get_osname (new_deployments->pdata[i]); + if (!g_str_equal (osname_0, osname_i)) + { + show_osname = TRUE; + break; + } + } + + for (guint i = 0; i < new_deployments->len; i++) + { + OstreeDeployment *deployment = new_deployments->pdata[i]; + if (!install_deployment_kernel (self, repo, new_bootversion, + deployment, new_deployments->len, + show_osname, cancellable, error)) + return FALSE; + } + + /* Create and swap bootlinks for *new* version */ + if (!create_new_bootlinks (self, new_bootversion, + new_deployments, + cancellable, error)) + return FALSE; + if (!swap_bootlinks (self, new_bootversion, new_deployments, + cancellable, error)) + return FALSE; + + g_debug ("Using bootloader: %s", bootloader ? + g_type_name (G_TYPE_FROM_INSTANCE (bootloader)) : "(none)"); + + if (bootloader) + { + if (!_ostree_bootloader_write_config (bootloader, new_bootversion, + cancellable, error)) + return glnx_prefix_error (error, "Bootloader write config"); + } + + if (!prepare_new_bootloader_link (self, self->bootversion, new_bootversion, + cancellable, error)) + return FALSE; + + if (!full_system_sync (self, out_syncstats, cancellable, error)) + return FALSE; + + if (!swap_bootloader (self, self->bootversion, new_bootversion, + cancellable, error)) + return FALSE; + + return TRUE; +} + /** * ostree_sysroot_write_deployments_with_options: * @self: Sysroot @@ -2054,10 +2142,6 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - gboolean boot_was_ro_mount = FALSE; - g_autoptr(OstreeBootloader) bootloader = NULL; - g_assert (self->loaded); /* Assign a bootserial to each new deployment. @@ -2096,51 +2180,38 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, deployment_root = ostree_sysroot_get_deployment_directory (self, deployment); if (!g_file_query_exists (deployment_root, NULL)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unable to find expected deployment root: %s", - gs_file_get_path_cached (deployment_root)); - goto out; - } + return glnx_throw (error, "Unable to find expected deployment root: %s", + gs_file_get_path_cached (deployment_root)); ostree_deployment_set_index (deployment, i); } if (self->booted_deployment && !found_booted_deployment) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Attempting to remove booted deployment"); - goto out; - } + return glnx_throw (error, "Attempting to remove booted deployment"); gboolean bootloader_is_atomic = FALSE; SyncStats syncstats = { 0, }; + g_autoptr(OstreeBootloader) bootloader = NULL; if (!requires_new_bootversion) { if (!create_new_bootlinks (self, self->bootversion, new_deployments, cancellable, error)) - goto out; + return FALSE; if (!full_system_sync (self, &syncstats, cancellable, error)) - { - g_prefix_error (error, "Full sync: "); - goto out; - } + return FALSE; if (!swap_bootlinks (self, self->bootversion, new_deployments, cancellable, error)) - goto out; + return FALSE; bootloader_is_atomic = TRUE; } else { - int new_bootversion = self->bootversion ? 0 : 1; - g_autofree char* new_loader_entries_dir = NULL; - g_autoptr(OstreeRepo) repo = NULL; - gboolean show_osname = FALSE; - + gboolean boot_was_ro_mount = FALSE; if (self->booted_deployment) boot_was_ro_mount = is_ro_mount ("/boot"); @@ -2148,95 +2219,33 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, if (boot_was_ro_mount) { + /* TODO: Use new mount namespace. https://github.com/ostreedev/ostree/issues/1265 */ if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) - { - glnx_set_prefix_error_from_errno (error, "%s", "Remounting /boot read-write"); - goto out; - } + return glnx_throw_errno_prefix (error, "Remounting /boot read-write"); } if (!_ostree_sysroot_query_bootloader (self, &bootloader, cancellable, error)) - goto out; + return FALSE; + bootloader_is_atomic = bootloader != NULL && _ostree_bootloader_is_atomic (bootloader); - new_loader_entries_dir = g_strdup_printf ("boot/loader.%d/entries", new_bootversion); - if (!glnx_shutil_rm_rf_at (self->sysroot_fd, new_loader_entries_dir, cancellable, error)) - goto out; - if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, new_loader_entries_dir, 0755, - cancellable, error)) - goto out; - - /* Need the repo to try and extract the versions for deployments. - * But this is a "nice-to-have" for the bootloader UI, so failure - * here is not fatal to the whole operation. We just gracefully - * fall back to the deployment index. */ - (void) ostree_sysroot_get_repo (self, &repo, cancellable, NULL); - - /* Only show the osname in bootloader titles if there are multiple - * osname's among the new deployments. Check for that here. */ - for (guint i = 1; i < new_deployments->len; i++) + /* Note equivalent of try/finally here */ + gboolean success = write_deployments_bootswap (self, new_deployments, opts, bootloader, + &syncstats, cancellable, error); + /* Below here don't set GError until the if (!success) check */ + if (boot_was_ro_mount) { - const char *osname_0 = ostree_deployment_get_osname (new_deployments->pdata[0]); - const char *osname_i = ostree_deployment_get_osname (new_deployments->pdata[i]); - if (!g_str_equal (osname_0, osname_i)) + if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_RDONLY | MS_SILENT, NULL) < 0) { - show_osname = TRUE; - break; + /* Only make this a warning because we don't want to + * completely bomb out if some other process happened to + * jump in and open a file there. See above TODO + * around doing this in a new mount namespace. + */ + g_printerr ("warning: Failed to remount /boot read-only: %s\n", strerror (errno)); } } - - for (guint i = 0; i < new_deployments->len; i++) - { - OstreeDeployment *deployment = new_deployments->pdata[i]; - if (!install_deployment_kernel (self, repo, new_bootversion, - deployment, new_deployments->len, - show_osname, cancellable, error)) - goto out; - } - - /* Create and swap bootlinks for *new* version */ - if (!create_new_bootlinks (self, new_bootversion, - new_deployments, - cancellable, error)) - goto out; - if (!swap_bootlinks (self, new_bootversion, new_deployments, - cancellable, error)) - goto out; - - g_debug ("Using bootloader: %s", bootloader ? - g_type_name (G_TYPE_FROM_INSTANCE (bootloader)) : "(none)"); - - if (bootloader) - bootloader_is_atomic = _ostree_bootloader_is_atomic (bootloader); - - if (bootloader) - { - if (!_ostree_bootloader_write_config (bootloader, new_bootversion, - cancellable, error)) - { - g_prefix_error (error, "Bootloader write config: "); - goto out; - } - } - - if (!prepare_new_bootloader_link (self, self->bootversion, new_bootversion, - cancellable, error)) - { - g_prefix_error (error, "Preparing final bootloader swap: "); - goto out; - } - - if (!full_system_sync (self, &syncstats, cancellable, error)) - { - g_prefix_error (error, "Full sync: "); - goto out; - } - - if (!swap_bootloader (self, self->bootversion, new_bootversion, - cancellable, error)) - { - g_prefix_error (error, "Final bootloader swap: "); - goto out; - } + if (!success) + return FALSE; } { g_autofree char *msg = @@ -2260,44 +2269,24 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, } if (!_ostree_sysroot_bump_mtime (self, error)) - goto out; + return FALSE; /* Now reload from disk */ if (!ostree_sysroot_load (self, cancellable, error)) - { - g_prefix_error (error, "Reloading deployments after commit: "); - goto out; - } + return glnx_prefix_error (error, "Reloading deployments after commit"); if (!cleanup_legacy_current_symlinks (self, cancellable, error)) - goto out; + return FALSE; /* And finally, cleanup of any leftover data. */ if (opts->do_postclean) { if (!ostree_sysroot_cleanup (self, cancellable, error)) - { - g_prefix_error (error, "Performing final cleanup: "); - goto out; - } + return glnx_prefix_error (error, "Performing final cleanup"); } - ret = TRUE; - out: - if (boot_was_ro_mount) - { - if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_RDONLY | MS_SILENT, NULL) < 0) - { - /* Only make this a warning because we don't want to - * completely bomb out if some other process happened to - * jump in and open a file there. - */ - int errsv = errno; - g_printerr ("warning: Failed to remount /boot read-only: %s\n", strerror (errsv)); - } - } - return ret; + return TRUE; } static gboolean From d379f87405d4d19a857d4977271d473acfc85250 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Mar 2018 13:23:25 -0400 Subject: [PATCH 06/80] tests/installed: Fix TESTS= being empty I broke this in https://github.com/ostreedev/ostree/pull/1509/commits/9b55aaea6f34b7094c44932a3c2e1cf2d54634fd I'd obviously tested *setting* it locally worked, but I didn't test that not having it set ran all the tests. I don't understand why we were doing the `+ ` pattern before; let's just check if it's empty. Closes: #1516 Approved by: jlebon --- tests/installed/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installed/run.sh b/tests/installed/run.sh index 38a9d769..8c7fe4d8 100755 --- a/tests/installed/run.sh +++ b/tests/installed/run.sh @@ -4,7 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) for tn in ${dn}/itest-*.sh; do - if [ -n "${TESTS+ }" ]; then + if [ -n "${TESTS:-}" ]; then tbn=$(basename "$tn" .sh) tbn=" ${tbn#itest-} " if [[ " $TESTS " != *$tbn* ]]; then From 925772eb09f45f463f9f321d89063973544cbf6d Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Mar 2018 15:56:13 -0700 Subject: [PATCH 07/80] tests: Fix unit test for ref-binding metadata The `ostree show` command is currently failing due to incorrect syntax, but we want to check that it fails because the metadata isn't there. Closes: #1520 Approved by: cgwalters --- tests/basic-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index b8e7eb07..e0ed2c32 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -851,7 +851,7 @@ $OSTREE show --print-metadata-key=ostree.ref-binding test2 > test2-ref-binding assert_file_has_content test2-ref-binding 'test2' $OSTREE commit ${COMMIT_ARGS} -b test2-unbound --no-bindings --tree=dir=${test_tmpdir}/checkout-test2 -if $OSTREE show --print-metadata-key=ostree.ref-binding; then +if $OSTREE show --print-metadata-key=ostree.ref-binding test2-unbound; then fatal "ref bindings found with --no-bindings?" fi echo "ok refbinding" From 3c14546017efaf6373cf9449cb076298c0e4e180 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Mar 2018 13:33:08 -0700 Subject: [PATCH 08/80] lib/repo-pull: Document all options in pull_from_remotes_async ostree_repo_pull_from_remotes_async() passes along some options to ostree_repo_pull_with_options(), so document them. Closes: #1519 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 89c67c8e..6170b011 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -5396,6 +5396,13 @@ copy_option (GVariantDict *master_options, * * `flags` (`i`): #OstreeRepoPullFlags to apply to the pull operation * * `inherit-transaction` (`b`): %TRUE to inherit an ongoing transaction on * the #OstreeRepo, rather than encapsulating the pull in a new one + * * `depth` (`i`): How far in the history to traverse; default is 0, -1 means infinite + * * `disable-static-deltas` (`b`): Do not use static deltas + * * `http-headers` (`a(ss)`): Additional headers to add to all HTTP requests + * * `subdirs` (`as`): Pull just these subdirectories + * * `update-frequency` (`u`): Frequency to call the async progress callback in + * milliseconds, if any; only values higher than 0 are valid + * * `append-user-agent` (`s`): Additional string to append to the user agent * * Since: 2017.8 */ From 2be4631738aaaad27d7689a6e6118c78659b5579 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Mar 2018 22:03:33 -0700 Subject: [PATCH 09/80] lib/commit: Fix a memory leak of OtChecksum Closes: #1521 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 16081a95..12c4103d 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1246,7 +1246,7 @@ write_metadata_object (OstreeRepo *self, } else { - OtChecksum checksum = { 0, }; + g_auto(OtChecksum) checksum = { 0, }; ot_checksum_init (&checksum); gsize len; const guint8*bufdata = g_bytes_get_data (buf, &len); From 9721be34e161ff5b2d699c52e1add633e8c33347 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Mar 2018 23:11:50 -0700 Subject: [PATCH 10/80] libotutil/checksum-utils: Fix memory management Ostree uses the OtChecksum data structure as a wrapper around GChecksum (depending on what libraries are available at compile time). According to the docs for g_checksum_get_digest(), a GChecksum value can no longer be updated after that function is called. Ostree enforces this by setting "initialized" to FALSE after getting the digest, but this leads to ot_checksum_clear() avoiding freeing any memory, leading to leaks. So this commit adds a "closed" value that gets set when getting a digest and checked when updating the value, so the initialized value can be used only for memory management. Closes: #1521 Approved by: jlebon --- src/libotutil/ot-checksum-utils.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c index 6e0e5641..6eb6fdc0 100644 --- a/src/libotutil/ot-checksum-utils.c +++ b/src/libotutil/ot-checksum-utils.c @@ -54,6 +54,7 @@ ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len) */ typedef struct { gboolean initialized; + gboolean closed; #if defined(HAVE_OPENSSL) EVP_MD_CTX *checksum; #elif defined(HAVE_GNUTLS) @@ -84,6 +85,7 @@ ot_checksum_init (OtChecksum *checksum) real->digest_len = g_checksum_type_get_length (G_CHECKSUM_SHA256); #endif g_assert_cmpint (real->digest_len, ==, _OSTREE_SHA256_DIGEST_LEN); + real->closed = FALSE; real->initialized = TRUE; } @@ -94,6 +96,7 @@ ot_checksum_update (OtChecksum *checksum, { OtRealChecksum *real = (OtRealChecksum*)checksum; g_return_if_fail (real->initialized); + g_return_if_fail (!real->closed); #if defined(HAVE_OPENSSL) g_assert (EVP_DigestUpdate (real->checksum, buf, len)); #elif defined(HAVE_GNUTLS) @@ -130,7 +133,7 @@ ot_checksum_get_digest (OtChecksum *checksum, { OtRealChecksum *real = (OtRealChecksum*)checksum; ot_checksum_get_digest_internal (real, buf, buflen); - real->initialized = FALSE; + real->closed = TRUE; } void @@ -143,7 +146,6 @@ ot_checksum_get_hexdigest (OtChecksum *checksum, guint8 digest_buf[digest_len]; ot_checksum_get_digest (checksum, digest_buf, digest_len); ot_bin2hex (buf, (guint8*)digest_buf, digest_len); - real->initialized = FALSE; } void From 28c7bc6d0e153a0b07bdb82d25473a490765067f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 29 Mar 2018 15:19:33 +0200 Subject: [PATCH 11/80] Don't write to parent repo In _try_clone_from_payload_link, don't try to do the clone in the parent repo, because we don't want to modify that. parent repos are typically used when you want a shared, immutable base. For example in flatpak, the parent repo is the system repo which you don't have write access to, so any modification to it will fail with EACCES, making it impossible to install via the system helper. Closes: #1524 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 12c4103d..7eb5ca93 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -777,8 +777,6 @@ _try_clone_from_payload_link (OstreeRepo *self, return TRUE; } } - if (self->parent_repo) - return _try_clone_from_payload_link (self->parent_repo, payload_checksum, file_info, tmpf, cancellable, error); return TRUE; } From 005d25cc75242658440ca40adffb0346790d569a Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 29 Mar 2018 12:44:16 -0700 Subject: [PATCH 12/80] lib: Fix a few comments Closes: #1526 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo-finder.c | 4 ++-- src/libostree/ostree-repo-pull.c | 2 +- src/libostree/ostree-repo.c | 10 +++++----- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 7eb5ca93..37f832c8 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2214,7 +2214,7 @@ metadata_size_valid (OstreeObjectType objtype, * @cancellable: Cancellable * @error: Error * - * Store the metadata object @variant. Return the checksum + * Store the metadata object @object. Return the checksum * as @out_csum. * * If @expected_checksum is not %NULL, verify it against the diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c index 829e8c6d..4cad81f9 100644 --- a/src/libostree/ostree-repo-finder.c +++ b/src/libostree/ostree-repo-finder.c @@ -132,8 +132,8 @@ static void resolve_cb (GObject *obj, * which the result provides. If the result provides the latest commit for a ref * across all of the results, the checksum will be set. Otherwise, if the * result provides an outdated commit, or doesn’t provide a given ref at all, - * the ref will not be set. Results which provide none of the requested @refs - * may be listed with an empty refs map. + * the checksum will not be set. Results which provide none of the requested + * @refs may be listed with an empty refs map. * * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given * @refs from those remotes. diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 6170b011..0eefbf7b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -5235,7 +5235,7 @@ find_remotes_cb (GObject *obj, for (i = 0; i < results->len; i++) { OstreeRepoFinderResult *result = g_ptr_array_index (results, i); - g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type utf8 utf8) */ + g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type OstreeCollectionRef utf8) */ gsize j, n_latest_refs; /* Previous error processing this result? */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8ff0d961..7d593f50 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5311,11 +5311,11 @@ summary_add_ref_entry (OstreeRepo *self, * `core/commit-update-summary` is set. * * If the `core/collection-id` key is set in the configuration, it will be - * included as %OSTREE_SUMMARY_COLLECTION_ID in the summary file. Refs from the - * `refs/mirrors` directory will be included in the generated summary file, - * listed under the %OSTREE_SUMMARY_COLLECTION_MAP key. Collection IDs and refs - * in %OSTREE_SUMMARY_COLLECTION_MAP are guaranteed to be in lexicographic - * order. + * included as %OSTREE_SUMMARY_COLLECTION_ID in the summary file. Refs that + * have associated collection IDs will be included in the generated summary + * file, listed under the %OSTREE_SUMMARY_COLLECTION_MAP key. Collection IDs + * and refs in %OSTREE_SUMMARY_COLLECTION_MAP are guaranteed to be in + * lexicographic order. */ gboolean ostree_repo_regenerate_summary (OstreeRepo *self, From 5d84f5e102e4d9d0cd99425b194d005fa1c10710 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Fri, 30 Mar 2018 18:11:45 -0700 Subject: [PATCH 13/80] lib/repo-pull: Rename a variable for clarity In libostree, the phrase "commit metadata" has two meanings-- one is the first dictionary in a commit GVariant that stores metadata such as ref bindings, and the other is the commit metadata in the summary file, which stores the commit size, checksum, and timestamp. In find_remotes_process_refs(), the entire commit GVariant was being referred to as commit metadata, so this commit changes the variable name and a comment to make things more consistent. Closes: #1528 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0eefbf7b..1205025e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4764,7 +4764,7 @@ find_remotes_process_refs (OstreeRepo *self, for (j = 0, n = g_variant_n_children (summary_refs); j < n; j++) { const guchar *csum_bytes; - g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL, stored_commit_metadata_v = NULL; + g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL, stored_commit_v = NULL; guint64 commit_size, commit_timestamp; gchar tmp_checksum[OSTREE_SHA256_STRING_LEN + 1]; gsize ref_index; @@ -4801,9 +4801,9 @@ find_remotes_process_refs (OstreeRepo *self, if (!collection_refv_contains (refs, summary_collection_id, ref_name, &ref_index)) continue; - /* Load the commit metadata from disk if possible, for verification. */ - if (!ostree_repo_load_commit (self, tmp_checksum, &stored_commit_metadata_v, NULL, NULL)) - stored_commit_metadata_v = NULL; + /* Load the commit from disk if possible, for verification. */ + if (!ostree_repo_load_commit (self, tmp_checksum, &stored_commit_v, NULL, NULL)) + stored_commit_v = NULL; /* Check the additional metadata. */ if (!g_variant_lookup (commit_metadata_v, OSTREE_COMMIT_TIMESTAMP, "t", &commit_timestamp)) @@ -4826,7 +4826,7 @@ find_remotes_process_refs (OstreeRepo *self, if (commit_metadata == NULL) { commit_metadata = commit_metadata_new (tmp_checksum, commit_size, - (stored_commit_metadata_v != NULL) ? ostree_commit_get_timestamp (stored_commit_metadata_v) : 0, + (stored_commit_v != NULL) ? ostree_commit_get_timestamp (stored_commit_v) : 0, NULL); g_hash_table_insert (commit_metadatas, commit_metadata->checksum, commit_metadata /* transfer */); From b190982ee62fecc6095b5c48e99a615a27687665 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 29 Mar 2018 00:03:43 -0700 Subject: [PATCH 14/80] lib/repo-pull: Improve error message when no summary is found In ostree_repo_remote_fetch_summary_with_options(), if no summary is found on the server and summary verification is enabled, the error message implies that it's the summary signature that's missing, which is misleading. This commit adds a more specific error message for the case of a missing summary, which has the side effect of explicitly checking for the case that signatures != NULL && summary == NULL after repo_remote_fetch_summary(), even though that should never happen. One effect of this is that if you run "flatpak remote-add" with an incorrect URL you get a more helpful error message, and similarly for other flatpak operations and other users of ostree. Closes: #1522 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 1205025e..37659459 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -5784,6 +5784,13 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) goto out; + if (gpg_verify_summary && summary == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)"); + goto out; + } + if (gpg_verify_summary && signatures == NULL) { g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, From 383e375099067d55cf49c864684c4c7dba2c5ac1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 3 Apr 2018 09:59:20 -0400 Subject: [PATCH 15/80] ci: Drop patches when building RPM Things fell over when downstream added a patch we had already merged upstream. Closes: #1530 Approved by: jlebon --- ci/libpaprci/Makefile.dist-packaging | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/libpaprci/Makefile.dist-packaging b/ci/libpaprci/Makefile.dist-packaging index 8ecaeffb..de77be6c 100644 --- a/ci/libpaprci/Makefile.dist-packaging +++ b/ci/libpaprci/Makefile.dist-packaging @@ -30,7 +30,7 @@ srpm: dist-snapshot origdir=$$(pwd); \ cd $(DISTGIT_NAME) && \ git stash && git pull -r && \ - sed -i -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(DISTGIT_NAME).spec && \ + sed -i -e '/^Patch/d' -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(DISTGIT_NAME).spec && \ rm -f *.src.rpm && \ $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec && mv *.src.rpm $${origdir}; \ fi From c4c2b5ebd155f9d17531db4ab9c2516699132745 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 3 Apr 2018 11:36:57 +0200 Subject: [PATCH 16/80] pull: Don't save summary to cache before validating signatures In case of some kind of race or other weirdness we might be getting non-matching versions of summary.sig and summary, where summary.sig is the latest version. Currently we're saving them to the cache directly after downloading them successfully, but they will then fail to gpg validate. Then on the next run we'll keep using the cached files even if they are incorrect, until summary.sig changes upstream. This changes the order so that we verify the signatures before saving to the cache, thus ensuring that we don't end up in a stuck state. Fixes https://github.com/ostreedev/ostree/issues/1523 Closes: #1529 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 47 ++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 37659459..f4661638 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2913,6 +2913,7 @@ repo_remote_fetch_summary (OstreeRepo *self, GVariant *options, GBytes **out_summary, GBytes **out_signatures, + gboolean *out_from_cache, GCancellable *cancellable, GError **error) { @@ -3015,32 +3016,13 @@ repo_remote_fetch_summary (OstreeRepo *self, goto out; } - if (!from_cache && *out_summary && *out_signatures) - { - g_autoptr(GError) temp_error = NULL; - - if (!_ostree_repo_cache_summary (self, - name, - *out_summary, - *out_signatures, - cancellable, - &temp_error)) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) - g_debug ("No permissions to save summary cache"); - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } - ret = TRUE; out: if (mainctx) g_main_context_pop_thread_default (mainctx); + + *out_from_cache = from_cache; return ret; } @@ -5763,6 +5745,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GBytes) signatures = NULL; gboolean ret = FALSE; gboolean gpg_verify_summary; + gboolean summary_is_from_cache; g_return_val_if_fail (OSTREE_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); @@ -5777,6 +5760,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, options, &summary, &signatures, + &summary_is_from_cache, cancellable, error)) goto out; @@ -5813,6 +5797,27 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, goto out; } + if (!summary_is_from_cache && summary && signatures) + { + g_autoptr(GError) temp_error = NULL; + + if (!_ostree_repo_cache_summary (self, + name, + summary, + signatures, + cancellable, + &temp_error)) + { + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) + g_debug ("No permissions to save summary cache"); + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + } + if (out_summary != NULL) *out_summary = g_steal_pointer (&summary); From cb3360fca656a478ba407c879f3cc675da78fe87 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Mar 2018 00:31:05 -0700 Subject: [PATCH 17/80] lib/repo: Add timestamps to OstreeRepoFinderResult Currently OstreeRepoFinderResult, a data structure used by pull code that supports P2P operations, has a hash table mapping refs to checksums but doesn't include timestamp information. This means that clients have no way of knowing just from the OstreeRepoFinderResult information if a commit being offered by a peer remote is an update or downgrade until they start pulling it. The client could check the summary or the commit metadata for the timestamps, but this requires adding the temporary remotes to the repo config, and ostree is already checking timestamps before returning the results, so I think it makes more sense for them to be returned rather than leaving it to the client. This limitation is especially important for offline computers, because for online computers the latest commit available from any remote is the latest commit, period. This commit adds a "ref_to_timestamp" hash table to OstreeRepoFinderResult that is symmetric to "ref_to_checksum" in that it shares the same keys. This is an API break, but it's part of the experimental API, and none of the current users of that (flatpak, eos-updater, and gnome-software) are affected. See the documentation for more details on "ref_to_timestamp". One thing to note is the data structure currently gets initialized in find_remotes_cb(), so only users of ostree_repo_find_remotes_async() will get them, not users of, say, ostree_repo_finder_resolve_all_async(). This is because the individual OstreeRepoFinder implementations don't currently access the timestamps (but I think this could be changed in the future if there's a need). This commit will allow P2P support to be added to flatpak_installation_list_installed_refs_for_update, which will allow GNOME Software to update apps from USB drives while offline (it's already possible online). Closes: #1518 Approved by: cgwalters --- src/libostree/ostree-repo-finder-avahi.c | 2 +- src/libostree/ostree-repo-finder-config.c | 2 +- src/libostree/ostree-repo-finder-mount.c | 2 +- src/libostree/ostree-repo-finder-override.c | 2 +- src/libostree/ostree-repo-finder.c | 8 +- src/libostree/ostree-repo-finder.h | 15 ++- src/libostree/ostree-repo-pull.c | 41 ++++++- tests/test-repo-finder-config.c | 126 ++++++++++++++++++++ 8 files changed, 189 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index 6687a835..514351fc 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -844,7 +844,7 @@ ostree_avahi_service_build_repo_finder_result (OstreeAvahiService } g_ptr_array_add (results, ostree_repo_finder_result_new (remote, OSTREE_REPO_FINDER (finder), - priority, supported_ref_to_checksum, + priority, supported_ref_to_checksum, NULL, GUINT64_FROM_BE (g_variant_get_uint64 (summary_timestamp)))); } } diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c index 76acb58e..5d1e1595 100644 --- a/src/libostree/ostree-repo-finder-config.c +++ b/src/libostree/ostree-repo-finder-config.c @@ -192,7 +192,7 @@ ostree_repo_finder_config_resolve_async (OstreeRepoFinder *find continue; } - g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0)); + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, NULL, 0)); } g_ptr_array_sort (results, results_compare_cb); diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 41a6bed2..7339fe52 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -545,7 +545,7 @@ G_GNUC_END_IGNORE_DEPRECATIONS * the code in ostree_repo_pull_from_remotes_async() will be able to * check it just as quickly as we can here; so don’t duplicate the * code. */ - g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0)); + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, NULL, 0)); } } diff --git a/src/libostree/ostree-repo-finder-override.c b/src/libostree/ostree-repo-finder-override.c index 5367708b..5bad9ace 100644 --- a/src/libostree/ostree-repo-finder-override.c +++ b/src/libostree/ostree-repo-finder-override.c @@ -243,7 +243,7 @@ ostree_repo_finder_override_resolve_async (OstreeRepoFinder *fi g_hash_table_iter_init (&iter, repo_remote_to_refs); while (g_hash_table_iter_next (&iter, (gpointer *) &remote, (gpointer *) &supported_ref_to_checksum)) - g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0)); + g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, NULL, 0)); g_ptr_array_sort (results, results_compare_cb); diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c index 4cad81f9..e7943c3e 100644 --- a/src/libostree/ostree-repo-finder.c +++ b/src/libostree/ostree-repo-finder.c @@ -436,6 +436,9 @@ G_DEFINE_BOXED_TYPE (OstreeRepoFinderResult, ostree_repo_finder_result, * priority * @ref_to_checksum: (element-type OstreeCollectionRef utf8) (transfer none): * map of collection–ref pairs to checksums provided by this result + * @ref_to_timestamp: (element-type OstreeCollectionRef guint64) (nullable) + * (transfer none): map of collection–ref pairs to timestamps provided by this + * result * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when * the summary file for the result was last modified, or `0` if this is unknown * @@ -450,6 +453,7 @@ ostree_repo_finder_result_new (OstreeRemote *remote, OstreeRepoFinder *finder, gint priority, GHashTable *ref_to_checksum, + GHashTable *ref_to_timestamp, guint64 summary_last_modified) { g_autoptr(OstreeRepoFinderResult) result = NULL; @@ -463,6 +467,7 @@ ostree_repo_finder_result_new (OstreeRemote *remote, result->finder = g_object_ref (finder); result->priority = priority; result->ref_to_checksum = g_hash_table_ref (ref_to_checksum); + result->ref_to_timestamp = ref_to_timestamp != NULL ? g_hash_table_ref (ref_to_timestamp) : NULL; result->summary_last_modified = summary_last_modified; return g_steal_pointer (&result); @@ -484,7 +489,7 @@ ostree_repo_finder_result_dup (OstreeRepoFinderResult *result) return ostree_repo_finder_result_new (result->remote, result->finder, result->priority, result->ref_to_checksum, - result->summary_last_modified); + result->ref_to_timestamp, result->summary_last_modified); } /** @@ -554,6 +559,7 @@ ostree_repo_finder_result_free (OstreeRepoFinderResult *result) /* This may be NULL iff the result is freed half-way through find_remotes_cb() * in ostree-repo-pull.c, and at no other time. */ g_clear_pointer (&result->ref_to_checksum, g_hash_table_unref); + g_clear_pointer (&result->ref_to_timestamp, g_hash_table_unref); g_object_unref (result->finder); ostree_remote_unref (result->remote); g_free (result); diff --git a/src/libostree/ostree-repo-finder.h b/src/libostree/ostree-repo-finder.h index bb1a437e..e622c9a6 100644 --- a/src/libostree/ostree-repo-finder.h +++ b/src/libostree/ostree-repo-finder.h @@ -99,6 +99,8 @@ GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, * @ref_to_checksum: (element-type OstreeCollectionRef utf8): map of collection–ref * pairs to checksums provided by this remote; values may be %NULL to * indicate this remote doesn’t provide that ref + * @ref_to_timestamp: (element-type OstreeCollectionRef guint64) (nullable): map of + * collection–ref pairs to timestamps; values may be 0 for various reasons * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when * the summary file on the remote was last modified, or `0` if unknown * @@ -122,6 +124,15 @@ GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, * should be available locally, so the details for each checksum can be looked * up using ostree_repo_load_commit(). * + * @ref_to_timestamp provides timestamps for the set of refs in + * @ref_to_checksum. The refs are keys (of type #OstreeCollectionRef) and the + * values are guint64 pointers with the timestamp associated with the checksum + * provided in @ref_to_checksum. @ref_to_timestamp can be %NULL, and when it's + * not, the timestamps are zero when any of the following conditions are met: + * (1) the override-commit-ids option was used on + * ostree_repo_find_remotes_async (2) there was an error in trying to get the + * commit metadata (3) the checksum for this ref is %NULL in @ref_to_checksum. + * * Since: 2017.8 */ typedef struct @@ -131,9 +142,10 @@ typedef struct gint priority; GHashTable *ref_to_checksum; guint64 summary_last_modified; + GHashTable *ref_to_timestamp; /*< private >*/ - gpointer padding[4]; + gpointer padding[3]; } OstreeRepoFinderResult; _OSTREE_PUBLIC @@ -144,6 +156,7 @@ OstreeRepoFinderResult *ostree_repo_finder_result_new (OstreeRemote *remote, OstreeRepoFinder *finder, gint priority, GHashTable *ref_to_checksum, + GHashTable *ref_to_timestamp, guint64 summary_last_modified); _OSTREE_PUBLIC OstreeRepoFinderResult *ostree_repo_finder_result_dup (OstreeRepoFinderResult *result); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f4661638..be7cb228 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4862,6 +4862,7 @@ find_remotes_cb (GObject *obj, g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */ g_autoptr(OstreeFetcher) fetcher = NULL; g_autofree const gchar **ref_to_latest_commit = NULL; /* indexed as @refs; (element-type commit-checksum) */ + g_autofree guint64 *ref_to_latest_timestamp = NULL; /* indexed as @refs; (element-type commit-timestamp) */ gsize n_refs; g_autofree char **override_commit_ids = NULL; g_autoptr(GPtrArray) remotes_to_remove = NULL; /* (element-type OstreeRemote) */ @@ -5017,6 +5018,7 @@ find_remotes_cb (GObject *obj, * it’s been moved to @refs_and_remotes_table and is now potentially out * of date. */ g_clear_pointer (&result->ref_to_checksum, g_hash_table_unref); + g_clear_pointer (&result->ref_to_timestamp, g_hash_table_unref); result->summary_last_modified = summary_last_modified; } @@ -5153,8 +5155,12 @@ find_remotes_cb (GObject *obj, * * @ref_to_latest_commit is indexed by @ref_index, and its values are the * latest checksum for each ref. If override-commit-ids was used, - * @ref_to_latest_commit won't be initialized or used.*/ + * @ref_to_latest_commit won't be initialized or used. + * + * @ref_to_latest_timestamp is also indexed by @ref_index, and its values are + * the latest timestamp for each ref, when available.*/ ref_to_latest_commit = g_new0 (const gchar *, n_refs); + ref_to_latest_timestamp = g_new0 (guint64, n_refs); for (i = 0; i < n_refs; i++) { @@ -5195,6 +5201,11 @@ find_remotes_cb (GObject *obj, * the summary or commit metadata files above. */ ref_to_latest_commit[i] = latest_checksum; + if (latest_checksum != NULL && latest_commit_metadata != NULL) + ref_to_latest_timestamp[i] = latest_commit_metadata->timestamp; + else + ref_to_latest_timestamp[i] = 0; + if (latest_commit_metadata != NULL) { latest_commit_timestamp_str = uint64_secs_to_iso8601 (latest_commit_metadata->timestamp); @@ -5218,6 +5229,7 @@ find_remotes_cb (GObject *obj, { OstreeRepoFinderResult *result = g_ptr_array_index (results, i); g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type OstreeCollectionRef utf8) */ + g_autoptr(GHashTable) validated_ref_to_timestamp = NULL; /* (element-type OstreeCollectionRef guint64) */ gsize j, n_latest_refs; /* Previous error processing this result? */ @@ -5231,11 +5243,24 @@ find_remotes_cb (GObject *obj, (GDestroyNotify) ostree_collection_ref_free, g_free); + validated_ref_to_timestamp = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); if (override_commit_ids) { for (j = 0; refs[j] != NULL; j++) - g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]), - g_strdup (override_commit_ids[j])); + { + guint64 *timestamp_ptr; + + g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]), + g_strdup (override_commit_ids[j])); + + timestamp_ptr = g_malloc (sizeof (guint64)); + *timestamp_ptr = 0; + g_hash_table_insert (validated_ref_to_timestamp, ostree_collection_ref_dup (refs[j]), + timestamp_ptr); + } } else { @@ -5244,6 +5269,7 @@ find_remotes_cb (GObject *obj, for (j = 0; refs[j] != NULL; j++) { const gchar *latest_commit_for_ref = ref_to_latest_commit[j]; + guint64 *timestamp_ptr; if (pointer_table_get (refs_and_remotes_table, j, i) != latest_commit_for_ref) latest_commit_for_ref = NULL; @@ -5252,6 +5278,14 @@ find_remotes_cb (GObject *obj, g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]), g_strdup (latest_commit_for_ref)); + + timestamp_ptr = g_malloc (sizeof (guint64)); + if (latest_commit_for_ref != NULL) + *timestamp_ptr = GUINT64_TO_BE (ref_to_latest_timestamp[j]); + else + *timestamp_ptr = 0; + g_hash_table_insert (validated_ref_to_timestamp, ostree_collection_ref_dup (refs[j]), + timestamp_ptr); } if (n_latest_refs == 0) @@ -5264,6 +5298,7 @@ find_remotes_cb (GObject *obj, } result->ref_to_checksum = g_steal_pointer (&validated_ref_to_checksum); + result->ref_to_timestamp = g_steal_pointer (&validated_ref_to_timestamp); g_ptr_array_add (final_results, g_steal_pointer (&g_ptr_array_index (results, i))); } diff --git a/tests/test-repo-finder-config.c b/tests/test-repo-finder-config.c index 61d49b48..a87e3f4b 100644 --- a/tests/test-repo-finder-config.c +++ b/tests/test-repo-finder-config.c @@ -303,6 +303,130 @@ test_repo_finder_config_mixed_configs (Fixture *fixture, g_main_context_pop_thread_default (context); } +/* Test that using ostree_repo_find_remotes_async() works too.*/ +static void +test_repo_finder_config_find_remotes (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinder) finder = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_auto(OstreeRepoFinderResultv) results = NULL; + g_autoptr(GError) error = NULL; + gsize i; + const OstreeCollectionRef ref0 = { "org.example.Collection0", "exampleos/x86_64/ref0" }; + const OstreeCollectionRef ref1 = { "org.example.Collection0", "exampleos/x86_64/ref1" }; + const OstreeCollectionRef ref2 = { "org.example.Collection1", "exampleos/x86_64/ref1" }; + const OstreeCollectionRef ref3 = { "org.example.Collection1", "exampleos/x86_64/ref2" }; + const OstreeCollectionRef ref4 = { "org.example.Collection2", "exampleos/x86_64/ref3" }; + const OstreeCollectionRef * const refs[] = { &ref0, &ref1, &ref2, &ref3, &ref4, NULL }; + OstreeRepoFinder *finders[2] = {NULL, }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + /* Put together various ref configuration files. */ + g_autofree gchar *collection0_uri = assert_create_remote (fixture, "org.example.Collection0", + "exampleos/x86_64/ref0", + "exampleos/x86_64/ref1", + NULL); + g_autofree gchar *collection1_uri = assert_create_remote (fixture, "org.example.Collection1", + "exampleos/x86_64/ref2", + NULL); + g_autofree gchar *no_collection_uri = assert_create_remote (fixture, NULL, + "exampleos/x86_64/ref3", + NULL); + + assert_create_remote_config (fixture->parent_repo, "remote0", collection0_uri, "org.example.Collection0"); + assert_create_remote_config (fixture->parent_repo, "remote1", collection1_uri, "org.example.Collection1"); + assert_create_remote_config (fixture->parent_repo, "remote0-copy", collection0_uri, "org.example.Collection0"); + assert_create_remote_config (fixture->parent_repo, "remote1-bad-copy", collection1_uri, "org.example.NotCollection1"); + assert_create_remote_config (fixture->parent_repo, "remote2", no_collection_uri, NULL); + + finders[0] = OSTREE_REPO_FINDER (ostree_repo_finder_config_new ()); + + /* Resolve the refs. */ + ostree_repo_find_remotes_async (fixture->parent_repo, refs, + NULL, finders, + NULL, NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_find_remotes_finish (fixture->parent_repo, + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (g_strv_length ((char **) results), ==, 3); + + /* Check that the results are correct: the invalid refs should have been + * ignored, and the valid results canonicalised and deduplicated. */ + for (i = 0; results[i] != NULL; i++) + { + const char *ref0_checksum, *ref1_checksum, *ref2_checksum, *ref3_checksum; + guint64 *ref0_timestamp, *ref1_timestamp, *ref2_timestamp, *ref3_timestamp; + + if (g_strcmp0 (ostree_remote_get_name (results[i]->remote), "remote0") == 0 || + g_strcmp0 (ostree_remote_get_name (results[i]->remote), "remote0-copy") == 0) + { + g_assert_cmpuint (g_hash_table_size (results[i]->ref_to_checksum), ==, 5); + + ref0_checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ref0); + g_assert_true (ostree_validate_checksum_string (ref0_checksum, NULL)); + + ref1_checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ref1); + g_assert_true (ostree_validate_checksum_string (ref1_checksum, NULL)); + + ref2_checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ref2); + g_assert (ref2_checksum == NULL); + + g_assert_cmpuint (g_hash_table_size (results[i]->ref_to_timestamp), ==, 5); + + ref0_timestamp = g_hash_table_lookup (results[i]->ref_to_timestamp, &ref0); + *ref0_timestamp = GUINT64_FROM_BE (*ref0_timestamp); + g_assert_cmpuint (*ref0_timestamp, >, 0); + + ref1_timestamp = g_hash_table_lookup (results[i]->ref_to_timestamp, &ref1); + *ref1_timestamp = GUINT64_FROM_BE (*ref1_timestamp); + g_assert_cmpuint (*ref1_timestamp, >, 0); + + ref2_timestamp = g_hash_table_lookup (results[i]->ref_to_timestamp, &ref2); + *ref2_timestamp = GUINT64_FROM_BE (*ref2_timestamp); + g_assert_cmpuint (*ref2_timestamp, ==, 0); + + g_assert_cmpstr (ostree_remote_get_url (results[i]->remote), ==, collection0_uri); + } + else if (g_strcmp0 (ostree_remote_get_name (results[i]->remote), "remote1") == 0) + { + g_assert_cmpuint (g_hash_table_size (results[i]->ref_to_checksum), ==, 5); + + ref3_checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ref3); + g_assert_true (ostree_validate_checksum_string (ref3_checksum, NULL)); + + ref0_checksum = g_hash_table_lookup (results[i]->ref_to_checksum, &ref0); + g_assert (ref0_checksum == NULL); + + g_assert_cmpuint (g_hash_table_size (results[i]->ref_to_timestamp), ==, 5); + + ref3_timestamp = g_hash_table_lookup (results[i]->ref_to_timestamp, &ref3); + *ref3_timestamp = GUINT64_FROM_BE (*ref3_timestamp); + g_assert_cmpuint (*ref3_timestamp, >, 0); + + ref0_timestamp = g_hash_table_lookup (results[i]->ref_to_timestamp, &ref0); + *ref0_timestamp = GUINT64_FROM_BE (*ref0_timestamp); + g_assert_cmpuint (*ref0_timestamp, ==, 0); + + g_assert_cmpstr (ostree_remote_get_url (results[i]->remote), ==, collection1_uri); + } + else + { + g_assert_not_reached (); + } + } + + g_main_context_pop_thread_default (context); +} + int main (int argc, char **argv) { setlocale (LC_ALL, ""); @@ -313,6 +437,8 @@ int main (int argc, char **argv) test_repo_finder_config_no_configs, teardown); g_test_add ("/repo-finder-config/mixed-configs", Fixture, NULL, setup, test_repo_finder_config_mixed_configs, teardown); + g_test_add ("/repo-finder-config/find-remotes", Fixture, NULL, setup, + test_repo_finder_config_find_remotes, teardown); return g_test_run(); } From 5215f24e683d65c354cdbb19767dc615b080900b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 16:40:35 -0400 Subject: [PATCH 18/80] tests: Merge installed/ and fedora-str/ directories Let's be opinionated now, and our installed/ test story *is* Ansible/STR. Merge `tests/fedora-str` into `tests/installed/`. Rework the nondestructive tests into a separate playbook run, and parallelize them for more efficiency. The destructive tests are also changed to use Ansible more. Add a higher level `run.sh` entrypoint and update the `README.md` with some useful tips. Closes: #1513 Approved by: jlebon --- .papr.yml | 25 +++++++----- ci/fah27-insttests.sh | 8 ++++ tests/fedora-str/README.md | 2 - tests/fedora-str/provision.sh | 8 ---- tests/fedora-str/sysinstall-tests.yml | 31 -------------- tests/installed/README.md | 17 +++++++- tests/installed/destructive.yml | 35 ++++++++++++++++ tests/installed/destructive/README.md | 5 +++ .../{ => destructive}/itest-bare-root.sh | 4 +- .../{ => destructive}/itest-deploy-selinux.sh | 4 +- .../{ => destructive}/itest-label-selinux.sh | 4 +- tests/installed/execute_batch.yml | 22 ++++++++++ tests/installed/libinsttest.sh | 2 +- tests/installed/nondestructive.yml | 40 +++++++++++++++++++ .../{ => nondestructive}/itest-bare-unit.sh | 6 ++- .../itest-bare-user-root.sh | 4 +- .../itest-bareuser-nouserxattrs.sh | 4 +- .../itest-payload-link.sh | 4 +- .../{ => nondestructive}/itest-pull-space.sh | 7 +++- .../{ => nondestructive}/itest-pull.sh | 20 +++++++--- .../{ => nondestructive}/itest-remotes.sh | 4 +- .../installed/nondestructive/libtest-core.sh | 1 + .../{fedora-str => installed}/overlay-git.yml | 2 +- .../{fedora-str => installed}/playbook-run.sh | 4 +- tests/installed/provision.sh | 12 ++++++ tests/installed/run.sh | 28 ++++++------- 26 files changed, 217 insertions(+), 86 deletions(-) create mode 100755 ci/fah27-insttests.sh delete mode 100644 tests/fedora-str/README.md delete mode 100755 tests/fedora-str/provision.sh delete mode 100644 tests/fedora-str/sysinstall-tests.yml create mode 100644 tests/installed/destructive.yml create mode 100644 tests/installed/destructive/README.md rename tests/installed/{ => destructive}/itest-bare-root.sh (97%) rename tests/installed/{ => destructive}/itest-deploy-selinux.sh (98%) rename tests/installed/{ => destructive}/itest-label-selinux.sh (98%) create mode 100644 tests/installed/execute_batch.yml create mode 100644 tests/installed/nondestructive.yml rename tests/installed/{ => nondestructive}/itest-bare-unit.sh (86%) rename tests/installed/{ => nondestructive}/itest-bare-user-root.sh (97%) rename tests/installed/{ => nondestructive}/itest-bareuser-nouserxattrs.sh (93%) rename tests/installed/{ => nondestructive}/itest-payload-link.sh (98%) rename tests/installed/{ => nondestructive}/itest-pull-space.sh (83%) rename tests/installed/{ => nondestructive}/itest-pull.sh (67%) rename tests/installed/{ => nondestructive}/itest-remotes.sh (88%) create mode 120000 tests/installed/nondestructive/libtest-core.sh rename tests/{fedora-str => installed}/overlay-git.yml (93%) rename tests/{fedora-str => installed}/playbook-run.sh (91%) create mode 100755 tests/installed/provision.sh diff --git a/.papr.yml b/.papr.yml index 501aeecf..7f1e2e49 100644 --- a/.papr.yml +++ b/.papr.yml @@ -7,24 +7,29 @@ branches: context: FAH27-insttests required: true -container: - image: registry.fedoraproject.org/fedora:27 +# FIXME; temporary workaround +# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162 +host: + distro: fedora/27/atomic + specs: + ram: 4096 +#container: +# image: registry.fedoraproject.org/fedora:27 tests: - - cd tests/fedora-str && ../../ci/build-rpm.sh - - ./tests/fedora-str/provision.sh - # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/ - - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest - - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/sysinstall-tests.yml + - cd /etc/yum.repos.d/ && curl -L -O https://copr.fedorainfracloud.org/coprs/walters/oci-kvm-hook/repo/fedora-27/walters-oci-kvm-hook-fedora-27.repo + - rpm-ostree install oci-kvm-hook && rpm-ostree ex livefs + - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/fah27-insttests.sh" artifacts: - - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log - - tests/fedora-str/artifacts/installed-tests.log + - tests/installed/artifacts/ --- # This suite skips the RPMs and does the build+unit tests in a container -inherit: true +inherit: false +container: + image: registry.fedoraproject.org/fedora:27 context: f27-primary env: # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain diff --git a/ci/fah27-insttests.sh b/ci/fah27-insttests.sh new file mode 100755 index 00000000..a045cf52 --- /dev/null +++ b/ci/fah27-insttests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/bash +set -xeuo pipefail + +./tests/installed/provision.sh +# TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/ +cd tests/installed +curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest +exec env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./run.sh diff --git a/tests/fedora-str/README.md b/tests/fedora-str/README.md deleted file mode 100644 index 8f219380..00000000 --- a/tests/fedora-str/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory holds tests that use the -[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface). diff --git a/tests/fedora-str/provision.sh b/tests/fedora-str/provision.sh deleted file mode 100755 index b8ecdb48..00000000 --- a/tests/fedora-str/provision.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/bash -set -xeuo pipefail - -dn=$(dirname $0) -. ${dn}/../../ci/libpaprci/libbuild.sh - -pkg_upgrade -pkg_install git rsync openssh-clients ansible standard-test-roles diff --git a/tests/fedora-str/sysinstall-tests.yml b/tests/fedora-str/sysinstall-tests.yml deleted file mode 100644 index 58f216ae..00000000 --- a/tests/fedora-str/sysinstall-tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -# This entrypoint right now just runs the sysinstalled-tests. ---- -- hosts: localhost - tags: - - atomic - remote_user: root - vars: - use_git_build: True - tests: "" - tasks: - - import_tasks: overlay-git.yml - when: use_git_build - # Next copy all of the tests/ directory - - name: Copy test data - synchronize: src=../../ dest=/root/tests/ archive=yes - # Down the line perhaps do each log file separately? - - name: Run shell script sysinstalled tests - shell: /root/tests/installed/run.sh &> /root/installed-tests.log - environment: - TESTS: "{{ tests }}" - register: sysinstalled_result - failed_when: False - - name: Fetch sysinstalled results - fetch: - src: /root/installed-tests.log - dest: artifacts/installed-tests.log - flat: yes - - name: Assert that sysinstalled tests succeeded - when: sysinstalled_result.rc != 0 - fail: - msg: "sysinstalled tests failed" diff --git a/tests/installed/README.md b/tests/installed/README.md index 45bf7d93..b3d14a74 100644 --- a/tests/installed/README.md +++ b/tests/installed/README.md @@ -1,2 +1,15 @@ -This suite of tests is currently run from redhat-ci; -they're intended to run as root. +This directory holds tests that use the +[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface). + +The high level structure is that we take a qcow2 file, inject +built RPMs into it, and then use Ansible to run tests. + +See `.papr.yml` for canonical usage. + +For local development, you should cache the qcow2 somewhere +stable (outside of this git repo). Also note that `../ci/build-rpms.sh` +does *not* pick up uncommitted changes! Stated more strongly, you +currently need to run `build-rpms.sh` after every change. + +To run just a specific test, use e.g.: +`env TEST_SUBJECTS=/path/to/qcow2 ./playbook-run.sh -e tests=.*pull nondestructive.yml` diff --git a/tests/installed/destructive.yml b/tests/installed/destructive.yml new file mode 100644 index 00000000..26843e7d --- /dev/null +++ b/tests/installed/destructive.yml @@ -0,0 +1,35 @@ +# This entrypoint right now just runs the sysinstalled-tests. +--- +- hosts: localhost + tags: + - atomic + remote_user: root + vars: + use_git_build: True + tests: "." + tasks: + - import_tasks: overlay-git.yml + when: use_git_build + # Next copy all of the tests/ directory + - name: Copy test data + synchronize: src=../../ dest=/root/tests/ archive=yes + - find: + paths: /root/tests/installed/destructive + patterns: "itest-*.sh" + register: all_tests + - set_fact: + selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}" + - assert: + that: + - "{{ selected_tests|length }} != 0" + - file: path=/root/logs state=directory + - block: + - name: Run destructive tests + shell: "{{ item }} &> /root/logs/$(basename {{ item }}).log" + with_items: + - "{{ selected_tests }}" + always: + - synchronize: + src: /root/logs/ + dest: artifacts/installed-destructive + mode: pull diff --git a/tests/installed/destructive/README.md b/tests/installed/destructive/README.md new file mode 100644 index 00000000..cafe605a --- /dev/null +++ b/tests/installed/destructive/README.md @@ -0,0 +1,5 @@ +This suite of tests is run from PAPR. Everything in here is destructive; it's +recommended to only run them in disposable virtual machines. This is done +in `tests/fedora-str/sysinstalled-tests.yml`, which currently uses a single VM +and runs the tests serially. It's likely in the future this will be changed +to do one VM per test. diff --git a/tests/installed/itest-bare-root.sh b/tests/installed/destructive/itest-bare-root.sh similarity index 97% rename from tests/installed/itest-bare-root.sh rename to tests/installed/destructive/itest-bare-root.sh index 0d384c2c..3a5302ee 100755 --- a/tests/installed/itest-bare-root.sh +++ b/tests/installed/destructive/itest-bare-root.sh @@ -5,9 +5,10 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh echo "1..2" +date cd /ostree/repo/tmp rm co -rf @@ -40,3 +41,4 @@ rm co -rf rm co-testref -rf echo "ok xattrs" +date diff --git a/tests/installed/itest-deploy-selinux.sh b/tests/installed/destructive/itest-deploy-selinux.sh similarity index 98% rename from tests/installed/itest-deploy-selinux.sh rename to tests/installed/destructive/itest-deploy-selinux.sh index 2ad34c3c..92f6993c 100755 --- a/tests/installed/itest-deploy-selinux.sh +++ b/tests/installed/destructive/itest-deploy-selinux.sh @@ -5,8 +5,9 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date # Create a new deployment ostree admin deploy --karg-proc-cmdline ${host_refspec} new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1 @@ -54,3 +55,4 @@ assert_file_has_content_literal bootlsz.txt 'system_u:object_r:boot_t:s0 initram ostree admin undeploy 0 ostree refs --delete test-label +date diff --git a/tests/installed/itest-label-selinux.sh b/tests/installed/destructive/itest-label-selinux.sh similarity index 98% rename from tests/installed/itest-label-selinux.sh rename to tests/installed/destructive/itest-label-selinux.sh index 463887a0..2a492858 100755 --- a/tests/installed/itest-label-selinux.sh +++ b/tests/installed/destructive/itest-label-selinux.sh @@ -5,8 +5,9 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date cd /ostree/repo/tmp rm co -rf ostree checkout -H ${host_refspec} co @@ -85,3 +86,4 @@ assert_streq "${oldcon}" "${newcon}" rm co -rf ostree refs --delete testbranch echo "ok checkout selinux and skip-list" +date diff --git a/tests/installed/execute_batch.yml b/tests/installed/execute_batch.yml new file mode 100644 index 00000000..7fd8374b --- /dev/null +++ b/tests/installed/execute_batch.yml @@ -0,0 +1,22 @@ +##################### +# execute_batch.yml +##################### +- name: Begin async command execution + shell: "{{ async_item }} &> {{ logdir }}/{{ async_item|basename }}.log" + # 10 minutes; the PAPR tester generally times out before that + async: 600 + poll: 0 + with_items: "{{ async_commands }}" + loop_control: + loop_var: "async_item" + register: async_results + +- name: Check async command status + async_status: + jid: "{{ async_result_item.ansible_job_id }}" + with_items: "{{ async_results.results }}" + loop_control: + loop_var: "async_result_item" + register: async_poll_results + until: async_poll_results.finished + retries: 240 diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh index 96da9545..4968adc0 100644 --- a/tests/installed/libinsttest.sh +++ b/tests/installed/libinsttest.sh @@ -20,7 +20,7 @@ # Boston, MA 02111-1307, USA. dn=$(dirname $0) -. ${dn}/libtest-core.sh +. ${dn}/../libtest-core.sh # Copy of bits from tap-test test_tmpdir= diff --git a/tests/installed/nondestructive.yml b/tests/installed/nondestructive.yml new file mode 100644 index 00000000..88a64874 --- /dev/null +++ b/tests/installed/nondestructive.yml @@ -0,0 +1,40 @@ +# Nondestructive sysinstalled tests, run in parallel. +--- +- hosts: localhost + tags: + - atomic + remote_user: root + vars: + use_git_build: True + tests: "." + # Arbitrary...we want some parallelism + batching_factor: 4 + tasks: + - import_tasks: overlay-git.yml + when: use_git_build + # Next copy all of the tests/ directory + - name: Copy test data + synchronize: src=../../ dest=/root/tests/ archive=yes + - find: + paths: /root/tests/installed/nondestructive + patterns: "itest-*.sh" + register: all_tests + - set_fact: + selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}" + - assert: + that: + - "{{ selected_tests|length }} != 0" + - file: path=/root/logs state=directory + - block: + - name: Run nondestructive tests + vars: + logdir: /root/logs + async_commands: "{{ item }}" + include_tasks: execute_batch.yml + with_items: + - "{{ selected_tests | batch('{{ batching_factor }}') | list }}" + always: + - synchronize: + src: /root/logs + dest: artifacts/installed-nondestructive + mode: pull diff --git a/tests/installed/itest-bare-unit.sh b/tests/installed/nondestructive/itest-bare-unit.sh similarity index 86% rename from tests/installed/itest-bare-unit.sh rename to tests/installed/nondestructive/itest-bare-unit.sh index fe07f24f..e3312608 100755 --- a/tests/installed/itest-bare-unit.sh +++ b/tests/installed/nondestructive/itest-bare-unit.sh @@ -6,16 +6,18 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date # These tests sort of bypass the installed-tests spec; # fixing that would require installing g-d-t-r, though # more ideally we architect things with a "control" container # distinct from the host. -export G_TEST_SRCDIR=$(realpath $dn/../..) +export G_TEST_SRCDIR=$(realpath $dn/../../..) # Use /var/tmp to hopefully use XFS + O_TMPFILE etc. prepare_tmpdir /var/tmp trap _tmpdir_cleanup EXIT /usr/libexec/installed-tests/libostree/test-basic.sh /usr/libexec/installed-tests/libostree/test-basic-c +date diff --git a/tests/installed/itest-bare-user-root.sh b/tests/installed/nondestructive/itest-bare-user-root.sh similarity index 97% rename from tests/installed/itest-bare-user-root.sh rename to tests/installed/nondestructive/itest-bare-user-root.sh index f5feac96..40722384 100755 --- a/tests/installed/itest-bare-user-root.sh +++ b/tests/installed/nondestructive/itest-bare-user-root.sh @@ -5,9 +5,10 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh echo "1..1" +date prepare_tmpdir ostree --repo=repo init --mode=bare-user @@ -37,3 +38,4 @@ ostree --repo=repo ls -X rootfs /usr/lib/dbus-daemon-helper >ls.txt assert_file_has_content ls.txt '^-007.. 0 81 .*security.selinux.*/usr/lib/dbus-daemon-helper' assert_not_file_has_content ls.txt 'user.ostreemeta' echo "ok bare-user link-checkout-speedup with modified xattrs maintains uids" +date diff --git a/tests/installed/itest-bareuser-nouserxattrs.sh b/tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh similarity index 93% rename from tests/installed/itest-bareuser-nouserxattrs.sh rename to tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh index c8c07948..7063286b 100755 --- a/tests/installed/itest-bareuser-nouserxattrs.sh +++ b/tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh @@ -8,10 +8,11 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh prepare_tmpdir trap _tmpdir_cleanup EXIT +date mkdir mnt mount -t tmpfs tmpfs mnt @@ -21,3 +22,4 @@ if ostree --repo=mnt/repo init --mode=bare-user 2>err.txt; then fi umount mnt assert_file_has_content err.txt "Operation not supported" +date diff --git a/tests/installed/itest-payload-link.sh b/tests/installed/nondestructive/itest-payload-link.sh similarity index 98% rename from tests/installed/itest-payload-link.sh rename to tests/installed/nondestructive/itest-payload-link.sh index f0576d82..62b84667 100755 --- a/tests/installed/itest-payload-link.sh +++ b/tests/installed/nondestructive/itest-payload-link.sh @@ -22,9 +22,10 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh echo "1..1" +date # Use /var/tmp so we have O_TMPFILE etc. prepare_tmpdir /var/tmp @@ -86,3 +87,4 @@ cat payload-links.txt | while read i; do done set -x echo "ok pull creates .payload-link" +date diff --git a/tests/installed/itest-pull-space.sh b/tests/installed/nondestructive/itest-pull-space.sh similarity index 83% rename from tests/installed/itest-pull-space.sh rename to tests/installed/nondestructive/itest-pull-space.sh index 8d218b15..925629b2 100755 --- a/tests/installed/itest-pull-space.sh +++ b/tests/installed/nondestructive/itest-pull-space.sh @@ -4,21 +4,24 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date prepare_tmpdir trap _tmpdir_cleanup EXIT cd ${test_tmpdir} -truncate -s 100MB testblk.img +truncate -s 20MB testblk.img blkdev=$(losetup --find --show $(pwd)/testblk.img) mkfs.xfs ${blkdev} mkdir mnt mount ${blkdev} mnt ostree --repo=mnt/repo init --mode=bare-user +echo 'fsync=false' >> mnt/repo/config if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then fatal "succeeded in doing a pull with no free space" fi assert_file_has_content err.txt "min-free-space-percent" umount mnt losetup -d ${blkdev} +date diff --git a/tests/installed/itest-pull.sh b/tests/installed/nondestructive/itest-pull.sh similarity index 67% rename from tests/installed/itest-pull.sh rename to tests/installed/nondestructive/itest-pull.sh index b80acdbc..a7cd922d 100755 --- a/tests/installed/itest-pull.sh +++ b/tests/installed/nondestructive/itest-pull.sh @@ -5,7 +5,8 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date prepare_tmpdir /var/tmp trap _tmpdir_cleanup EXIT @@ -15,14 +16,19 @@ mkdir repo ostree --repo=repo init --mode=archive echo -e '[archive]\nzlib-level=1\n' >> repo/config host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,') -ostree --repo=repo pull-local /ostree/repo ${host_commit} +log_timestamps() { + date + "$@" + date +} +log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit} ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref} run_tmp_webserver $(pwd)/repo # Now test pulling via HTTP (no deltas) to a new bare-user repo ostree --repo=bare-repo init --mode=bare-user ostree --repo=bare-repo remote add origin --set=gpg-verify=false $(cat ${test_tmpdir}/httpd-address) -ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref} +log_timestamps ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref} rm bare-repo repo -rf @@ -33,7 +39,11 @@ mkdir tmpfs mnt mount --bind tmpfs mnt cd mnt ostree --repo=repo init --mode=bare -ostree --repo=repo pull-local /ostree/repo ${host_commit} -ostree --repo=repo fsck +log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit} +log_timestamps ostree --repo=repo fsck cd .. umount mnt + +kill -TERM $(cat ${test_tmpdir}/httpd-pid) +echo "ok" +date diff --git a/tests/installed/itest-remotes.sh b/tests/installed/nondestructive/itest-remotes.sh similarity index 88% rename from tests/installed/itest-remotes.sh rename to tests/installed/nondestructive/itest-remotes.sh index 8a79015f..836e35ad 100755 --- a/tests/installed/itest-remotes.sh +++ b/tests/installed/nondestructive/itest-remotes.sh @@ -5,7 +5,8 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libinsttest.sh +. ${dn}/../libinsttest.sh +date prepare_tmpdir trap _tmpdir_cleanup EXIT @@ -14,3 +15,4 @@ ostree remote list > remotes.txt if ! test -s remotes.txt; then assert_not_reached "no ostree remotes" fi +date diff --git a/tests/installed/nondestructive/libtest-core.sh b/tests/installed/nondestructive/libtest-core.sh new file mode 120000 index 00000000..d26203e2 --- /dev/null +++ b/tests/installed/nondestructive/libtest-core.sh @@ -0,0 +1 @@ +../libtest-core.sh \ No newline at end of file diff --git a/tests/fedora-str/overlay-git.yml b/tests/installed/overlay-git.yml similarity index 93% rename from tests/fedora-str/overlay-git.yml rename to tests/installed/overlay-git.yml index 65fe9379..0018a1ba 100644 --- a/tests/fedora-str/overlay-git.yml +++ b/tests/installed/overlay-git.yml @@ -6,7 +6,7 @@ - set_fact: ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}" - name: Copy locally built RPMs - synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes + synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes - shell: ostree admin unlock || true # Install the RPMs we already have. For the test suite we use rpm2cpio # since it depends on libsoup, but we're not using that yet for the sysinstalled tests diff --git a/tests/fedora-str/playbook-run.sh b/tests/installed/playbook-run.sh similarity index 91% rename from tests/fedora-str/playbook-run.sh rename to tests/installed/playbook-run.sh index 96e438d5..c1887297 100755 --- a/tests/fedora-str/playbook-run.sh +++ b/tests/installed/playbook-run.sh @@ -17,7 +17,9 @@ somewhere persistent. EOF exit 1 fi -ls -al ${TEST_SUBJECTS} +for subj in ${TEST_SUBJECTS}; do + ls -al ${subj} && file ${subj} +done # This is required rpm -q standard-test-roles diff --git a/tests/installed/provision.sh b/tests/installed/provision.sh new file mode 100755 index 00000000..3a1efe94 --- /dev/null +++ b/tests/installed/provision.sh @@ -0,0 +1,12 @@ +#!/usr/bin/bash +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/../../ci/libpaprci/libbuild.sh + +pkg_upgrade +pkg_install git rsync openssh-clients ansible standard-test-roles + +# "Hot patch" this to pick up https://pagure.io/standard-test-roles/pull-request/152 +# so we get parallelism +cd /usr/share/ansible/inventory && curl -L -O https://pagure.io/standard-test-roles/raw/master/f/inventory/standard-inventory-qcow2 diff --git a/tests/installed/run.sh b/tests/installed/run.sh index 8c7fe4d8..497777a7 100755 --- a/tests/installed/run.sh +++ b/tests/installed/run.sh @@ -1,17 +1,17 @@ -#!/bin/bash - +#!/usr/bin/bash +# Run all installed tests; see README.md in this directory for more +# information. set -xeuo pipefail -dn=$(dirname $0) -for tn in ${dn}/itest-*.sh; do - if [ -n "${TESTS:-}" ]; then - tbn=$(basename "$tn" .sh) - tbn=" ${tbn#itest-} " - if [[ " $TESTS " != *$tbn* ]]; then - echo "Skipping: ${tn}" - continue - fi - fi - echo Executing: ${tn} - ${tn} +dn=$(cd $(dirname $0) && pwd) + +if ! test -d build; then + mkdir -p build + (cd build && ${dn}/../../ci/build-rpm.sh) +fi + +# TODO: parallelize this +PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive.yml} +for playbook in $PLAYBOOKS; do + time ${dn}/playbook-run.sh -v ${dn}/${playbook} done From 9d1dacfcc8f95720bc4b42d7ef0da33f0e6b665f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 7 Apr 2018 11:40:53 -0400 Subject: [PATCH 19/80] bin: Hide `admin instutil` command Add a "hidden command" flag, and use it for `admin instutil` since I regret adding it, and people should be using the API. Prep for adding another hidden command as part of staging deployments. (Down the line we should investigate deduplicating the recursive command parsing code) Closes: #1535 Approved by: jlebon --- src/ostree/ot-admin-builtin-instutil.c | 9 ++++++--- src/ostree/ot-builtin-admin.c | 13 ++++++++----- src/ostree/ot-builtin-remote.c | 10 ++++++---- src/ostree/ot-builtin-static-delta.c | 3 ++- src/ostree/ot-main.c | 9 ++++++--- src/ostree/ot-main.h | 3 ++- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/ostree/ot-admin-builtin-instutil.c b/src/ostree/ot-admin-builtin-instutil.c index 035bd4f8..fe0d80a4 100644 --- a/src/ostree/ot-admin-builtin-instutil.c +++ b/src/ostree/ot-admin-builtin-instutil.c @@ -55,9 +55,12 @@ ostree_admin_instutil_option_context_new_with_commands (void) while (command->name != NULL) { - g_string_append_printf (summary, "\n %-24s", command->name); - if (command->description != NULL) - g_string_append_printf (summary, "%s", command->description); + if ((command->flags & OSTREE_BUILTIN_FLAG_HIDDEN) == 0) + { + g_string_append_printf (summary, "\n %-24s", command->name); + if (command->description != NULL) + g_string_append_printf (summary, "%s", command->description); + } command++; } diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index fd6d9a8f..1262c5a5 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -45,9 +45,9 @@ static OstreeCommand admin_subcommands[] = { { "init-fs", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_init_fs, "Initialize a root filesystem" }, - { "instutil", OSTREE_BUILTIN_FLAG_NO_REPO, + { "instutil", OSTREE_BUILTIN_FLAG_NO_REPO | OSTREE_BUILTIN_FLAG_HIDDEN, ot_admin_builtin_instutil, - "Provide instutil commands, allow admin to change boot configuration and relabel selinux " }, + "Deprecated commands intended for installer programs" }, { "os-init", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_os_init, "Initialize empty state for given operating system" }, @@ -85,9 +85,12 @@ ostree_admin_option_context_new_with_commands (void) while (command->name != NULL) { - g_string_append_printf (summary, "\n %-19s", command->name); - if (command->description != NULL) - g_string_append_printf (summary, "%s", command->description); + if ((command->flags & OSTREE_BUILTIN_FLAG_HIDDEN) == 0) + { + g_string_append_printf (summary, "\n %-19s", command->name); + if (command->description != NULL) + g_string_append_printf (summary, "%s", command->description); + } command++; } diff --git a/src/ostree/ot-builtin-remote.c b/src/ostree/ot-builtin-remote.c index 4deee8c6..0712d5bf 100644 --- a/src/ostree/ot-builtin-remote.c +++ b/src/ostree/ot-builtin-remote.c @@ -73,10 +73,12 @@ remote_option_context_new_with_commands (void) while (subcommand->name != NULL) { - g_string_append_printf (summary, "\n %-18s", subcommand->name); - if (subcommand->description != NULL) - g_string_append_printf (summary, "%s", subcommand->description); - + if ((subcommand->flags & OSTREE_BUILTIN_FLAG_HIDDEN) == 0) + { + g_string_append_printf (summary, "\n %-18s", subcommand->name); + if (subcommand->description != NULL) + g_string_append_printf (summary, "%s", subcommand->description); + } subcommand++; } diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index d2a343f2..4f9ff2b2 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -117,7 +117,8 @@ static_delta_usage (char **argv, while (command->name) { - print_func (" %-17s%s\n", command->name, command->description ?: ""); + if ((command->flags & OSTREE_BUILTIN_FLAG_HIDDEN) == 0) + print_func (" %-17s%s\n", command->name, command->description ?: ""); command++; } diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 654a5f3d..237427a2 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -66,10 +66,13 @@ ostree_option_context_new_with_commands (OstreeCommand *commands) while (commands->name != NULL) { - g_string_append_printf (summary, "\n %-18s", commands->name); + if ((commands->flags & OSTREE_BUILTIN_FLAG_HIDDEN) == 0) + { + g_string_append_printf (summary, "\n %-18s", commands->name); - if (commands->description != NULL ) - g_string_append_printf (summary, "%s", commands->description); + if (commands->description != NULL ) + g_string_append_printf (summary, "%s", commands->description); + } commands++; } diff --git a/src/ostree/ot-main.h b/src/ostree/ot-main.h index ac75118c..b1b994b6 100644 --- a/src/ostree/ot-main.h +++ b/src/ostree/ot-main.h @@ -29,7 +29,8 @@ typedef enum { OSTREE_BUILTIN_FLAG_NONE = 0, OSTREE_BUILTIN_FLAG_NO_REPO = 1 << 0, - OSTREE_BUILTIN_FLAG_NO_CHECK = 1 << 1 + OSTREE_BUILTIN_FLAG_NO_CHECK = 1 << 1, + OSTREE_BUILTIN_FLAG_HIDDEN = 1 << 2, } OstreeBuiltinFlags; typedef enum { From ab61f812c4a9d21cae4290f3972f6769cb9773b1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 Apr 2018 13:27:15 -0400 Subject: [PATCH 20/80] tests/installed: Move auto-build logic to playbook-run.sh This makes it a bit more convenient to make a code change, then `rm -rf build && ./playbook-run.sh ...`. Closes: #1535 Approved by: jlebon --- tests/installed/playbook-run.sh | 6 ++++++ tests/installed/run.sh | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/installed/playbook-run.sh b/tests/installed/playbook-run.sh index c1887297..3136899f 100755 --- a/tests/installed/playbook-run.sh +++ b/tests/installed/playbook-run.sh @@ -3,6 +3,12 @@ # TEST_SUBJECTS being set. set -xeuo pipefail +dn=$(cd $(dirname $0) && pwd) +if ! test -d build; then + mkdir -p build + (cd build && ${dn}/../../ci/build-rpm.sh) +fi + # https://fedoraproject.org/wiki/CI/Tests if test -z "${TEST_SUBJECTS:-}"; then cat < Date: Wed, 11 Apr 2018 13:30:32 -0400 Subject: [PATCH 21/80] sysroot: Split out a helper function to delete a deployment dir Prep for staged deployments. Closes: #1535 Approved by: jlebon --- src/libostree/ostree-sysroot-cleanup.c | 70 +++++++++++++++----------- src/libostree/ostree-sysroot-private.h | 6 +++ 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 2e7cd44b..1d46222b 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -239,6 +239,44 @@ cleanup_other_bootversions (OstreeSysroot *self, return TRUE; } +/* Delete a deployment directory */ +gboolean +_ostree_sysroot_rmrf_deployment (OstreeSysroot *self, + OstreeDeployment *deployment, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *origin_relpath = ostree_deployment_get_origin_relpath (deployment); + g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); + struct stat stbuf; + glnx_autofd int deployment_fd = -1; + + if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE, + &deployment_fd, error)) + return FALSE; + + if (!glnx_fstat (deployment_fd, &stbuf, error)) + return FALSE; + + /* This shouldn't happen, because higher levels should + * disallow having the booted deployment not in the active + * deployment list, but let's be extra safe. */ + if (stbuf.st_dev == self->root_device && + stbuf.st_ino == self->root_inode) + return TRUE; + + /* This deployment wasn't referenced, so delete it */ + if (!_ostree_linuxfs_fd_alter_immutable_flag (deployment_fd, FALSE, + cancellable, error)) + return FALSE; + if (!glnx_shutil_rm_rf_at (self->sysroot_fd, origin_relpath, cancellable, error)) + return FALSE; + if (!glnx_shutil_rm_rf_at (self->sysroot_fd, deployment_path, cancellable, error)) + return FALSE; + + return TRUE; +} + /* As the bootloader configuration changes, we will have leftover deployments * on disk. This function deletes all deployments which aren't actively * referenced. @@ -279,36 +317,12 @@ cleanup_old_deployments (OstreeSysroot *self, { OstreeDeployment *deployment = all_deployment_dirs->pdata[i]; g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); - g_autofree char *origin_relpath = ostree_deployment_get_origin_relpath (deployment); - if (!g_hash_table_lookup (active_deployment_dirs, deployment_path)) - { - struct stat stbuf; - glnx_autofd int deployment_fd = -1; + if (g_hash_table_lookup (active_deployment_dirs, deployment_path)) + continue; - if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE, - &deployment_fd, error)) - return FALSE; - - if (!glnx_fstat (deployment_fd, &stbuf, error)) - return FALSE; - - /* This shouldn't happen, because higher levels should - * disallow having the booted deployment not in the active - * deployment list, but let's be extra safe. */ - if (stbuf.st_dev == root_stbuf.st_dev && - stbuf.st_ino == root_stbuf.st_ino) - continue; - - /* This deployment wasn't referenced, so delete it */ - if (!_ostree_linuxfs_fd_alter_immutable_flag (deployment_fd, FALSE, - cancellable, error)) - return FALSE; - if (!glnx_shutil_rm_rf_at (self->sysroot_fd, origin_relpath, cancellable, error)) - return FALSE; - if (!glnx_shutil_rm_rf_at (self->sysroot_fd, deployment_path, cancellable, error)) - return FALSE; - } + if (!_ostree_sysroot_rmrf_deployment (self, deployment, cancellable, error)) + return FALSE; } /* Clean up boot directories */ diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index b2776e8d..01b370e8 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -112,6 +112,12 @@ _ostree_sysroot_get_origin_relpath (GFile *path, GCancellable *cancellable, GError **error); +gboolean +_ostree_sysroot_rmrf_deployment (OstreeSysroot *sysroot, + OstreeDeployment *deployment, + GCancellable *cancellable, + GError **error); + char *_ostree_sysroot_join_lines (GPtrArray *lines); gboolean _ostree_sysroot_query_bootloader (OstreeSysroot *sysroot, From 7357d346c40a7eb57edccbd1bc29c6d324e14044 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 Apr 2018 13:31:18 -0400 Subject: [PATCH 22/80] tests: Better error message if target is not a symlink I broke the code for this and ended up adding this to debug it. Closes: #1535 Approved by: jlebon --- tests/admin-test.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index b546e142..6e3fd782 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -147,8 +147,13 @@ ln -s /ENOENT sysroot/ostree/deploy/testos/deploy/${rev}.3/etc/a-new-broken-syml ${CMD_PREFIX} ostree admin deploy --retain --os=testos testos:testos/buildmaster/x86_64-runtime assert_not_has_dir sysroot/boot/loader.0 assert_has_dir sysroot/boot/loader.1 -linktarget=$(readlink sysroot/ostree/deploy/testos/deploy/${rev}.4/etc/a-new-broken-symlink) -test "${linktarget}" = /ENOENT +link=sysroot/ostree/deploy/testos/deploy/${rev}.4/etc/a-new-broken-symlink +if ! test -L ${link}; then + ls -al ${link} + fatal "Not a symlink: ${link}" +fi +linktarget=$(readlink ${link}) +assert_streq "${linktarget}" /ENOENT assert_file_has_content sysroot/ostree/deploy/testos/deploy/${rev}.3/etc/os-release 'NAME=TestOS' assert_file_has_content sysroot/ostree/deploy/testos/deploy/${rev}.4/etc/os-release 'NAME=TestOS' assert_file_has_content sysroot/ostree/deploy/testos/deploy/${rev}.4/etc/a-new-config-file 'a new local config file' From b9fc3eaa154774531ae66835387615bb7a650bf2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 Apr 2018 13:33:31 -0400 Subject: [PATCH 23/80] tests/installed: Move tasks into tasks/ directory It's cleaner to separate test playbooks from their tasks. Closes: #1535 Approved by: jlebon --- tests/installed/destructive.yml | 5 ++++- tests/installed/nondestructive.yml | 3 ++- tests/installed/{ => tasks}/overlay-git.yml | 0 tests/installed/tasks/query-host.yml | 5 +++++ 4 files changed, 11 insertions(+), 2 deletions(-) rename tests/installed/{ => tasks}/overlay-git.yml (100%) create mode 100644 tests/installed/tasks/query-host.yml diff --git a/tests/installed/destructive.yml b/tests/installed/destructive.yml index 26843e7d..9529c7e9 100644 --- a/tests/installed/destructive.yml +++ b/tests/installed/destructive.yml @@ -8,7 +8,10 @@ use_git_build: True tests: "." tasks: - - import_tasks: overlay-git.yml + - import_tasks: tasks/query-host.yml + - set_fact: + rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}" + - import_tasks: tasks/overlay-git.yml when: use_git_build # Next copy all of the tests/ directory - name: Copy test data diff --git a/tests/installed/nondestructive.yml b/tests/installed/nondestructive.yml index 88a64874..dcd0d442 100644 --- a/tests/installed/nondestructive.yml +++ b/tests/installed/nondestructive.yml @@ -10,7 +10,8 @@ # Arbitrary...we want some parallelism batching_factor: 4 tasks: - - import_tasks: overlay-git.yml + - import_tasks: tasks/query-host.yml + - import_tasks: tasks/overlay-git.yml when: use_git_build # Next copy all of the tests/ directory - name: Copy test data diff --git a/tests/installed/overlay-git.yml b/tests/installed/tasks/overlay-git.yml similarity index 100% rename from tests/installed/overlay-git.yml rename to tests/installed/tasks/overlay-git.yml diff --git a/tests/installed/tasks/query-host.yml b/tests/installed/tasks/query-host.yml new file mode 100644 index 00000000..2a67dbdf --- /dev/null +++ b/tests/installed/tasks/query-host.yml @@ -0,0 +1,5 @@ +- command: rpm-ostree status --json + changed_when: False + register: rpmostree_status_json +- set_fact: + rpmostree_status: "{{ rpmostree_status_json.stdout | from_json }}" From 9199237b01687e1e0842f3926fd01b695014df2b Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Wed, 11 Apr 2018 18:06:39 -0400 Subject: [PATCH 24/80] Don't scan uncompressed_objects_dir if it doesn't exist A newly created archive-mode repository won't have a uncompressed-objects-cache directory, and uncompressed_objects_dir is -1 to flag that. The special meaning of -1 meaning "cwd" for libglnx means that the current directory was scanned as if it was an objects directory, producing unexpected results, especially if there were any two-letter files/subdirs in the current directory. Closes: #1537 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 37f832c8..5fd97902 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1438,7 +1438,8 @@ scan_loose_devino (OstreeRepo *self, return FALSE; } - if (self->mode == OSTREE_REPO_MODE_ARCHIVE) + if (self->mode == OSTREE_REPO_MODE_ARCHIVE && + self->uncompressed_objects_dir_fd != -1) { if (!scan_one_loose_devino (self, self->uncompressed_objects_dir_fd, devino_cache, cancellable, error)) From ff50495f67a95c9b2fef5f9b84bc91469a46eb27 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 11 Apr 2018 16:45:40 -0400 Subject: [PATCH 25/80] ci: turn gating back on for most testsuites Somehow, this slipped through in #1513. We weren't inheriting anymore, so `branches` defaulted back to just `master`. This means we weren't gating on most of the containerized builds anymore. Ouch! It'd make sense to teach PAPR to allow some defaults across all testsuites, even in the `inherit: false` case. Though it's tempting to also just change the hardcoded PAPR default to those branches since our use of Homu + PAPR at this point is pretty ubiquitous, and it doesn't really hurt for the ones that don't use it. Closes: #1536 Approved by: cgwalters --- .papr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.papr.yml b/.papr.yml index 7f1e2e49..a1a6741f 100644 --- a/.papr.yml +++ b/.papr.yml @@ -28,6 +28,10 @@ artifacts: # This suite skips the RPMs and does the build+unit tests in a container inherit: false +branches: + - master + - auto + - try container: image: registry.fedoraproject.org/fedora:27 context: f27-primary From eb506c759c666af2461f1ba3dda4e31ea49ebc41 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Feb 2018 15:27:59 -0500 Subject: [PATCH 26/80] Add concept of "staged" deployment Add API to write a deployment state to `/run/ostree/staged-deployment`, along with a systemd service which runs at shutdown time. This is a big change to the ostree model for hosts, but it closes a longstanding set of bugs; many, many people have hit the "losing changes in /etc" problem. It also avoids the other problem of racing with programs that modify `/etc` such as LVM backups: https://bugzilla.redhat.com/show_bug.cgi?id=1365297 We need this in particular to go to a full-on model for automatically updated host systems where (like a dual-partition model) everything is fully prepared and the reboot can be taken asynchronously. Closes: https://github.com/ostreedev/ostree/issues/545 Closes: #1503 Approved by: jlebon --- Makefile-boot.am | 3 +- Makefile-ostree.am | 1 + apidoc/ostree-sections.txt | 3 + src/boot/ostree-finalize-staged.service | 36 ++ src/libostree/libostree-devel.sym | 3 + src/libostree/ostree-cmdprivate.c | 5 +- src/libostree/ostree-cmdprivate.h | 1 + src/libostree/ostree-deployment-private.h | 2 + src/libostree/ostree-deployment.c | 13 + src/libostree/ostree-deployment.h | 3 +- src/libostree/ostree-sysroot-cleanup.c | 9 + src/libostree/ostree-sysroot-deploy.c | 415 ++++++++++++++---- src/libostree/ostree-sysroot-private.h | 21 + src/libostree/ostree-sysroot.c | 98 ++++- src/libostree/ostree-sysroot.h | 13 + src/ostree/ot-admin-builtin-deploy.c | 56 ++- src/ostree/ot-admin-builtin-finalize-staged.c | 58 +++ src/ostree/ot-admin-builtin-status.c | 14 +- src/ostree/ot-admin-builtins.h | 1 + src/ostree/ot-builtin-admin.c | 3 + tests/installed/destructive.yml | 4 + tests/installed/destructive/staged-deploy.yml | 24 + tests/installed/tasks/reboot.yml | 71 +++ 23 files changed, 747 insertions(+), 110 deletions(-) create mode 100644 src/boot/ostree-finalize-staged.service create mode 100644 src/ostree/ot-admin-builtin-finalize-staged.c create mode 100644 tests/installed/destructive/staged-deploy.yml create mode 100644 tests/installed/tasks/reboot.yml diff --git a/Makefile-boot.am b/Makefile-boot.am index d3d2f673..5b512b6c 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -39,7 +39,7 @@ endif if BUILDOPT_SYSTEMD systemdsystemunit_DATA = src/boot/ostree-prepare-root.service \ - src/boot/ostree-remount.service + src/boot/ostree-remount.service src/boot/ostree-finalize-staged.service systemdtmpfilesdir = $(prefix)/lib/tmpfiles.d dist_systemdtmpfiles_DATA = src/boot/ostree-tmpfiles.conf @@ -65,6 +65,7 @@ EXTRA_DIST += src/boot/dracut/module-setup.sh \ src/boot/mkinitcpio/ostree \ src/boot/ostree-prepare-root.service \ src/boot/ostree-remount.service \ + src/boot/ostree-finalize-staged.service \ src/boot/grub2/grub2-15_ostree \ src/boot/grub2/ostree-grub-generator \ $(NULL) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index cccbe300..bdd51a72 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -67,6 +67,7 @@ ostree_SOURCES += \ src/ostree/ot-admin-builtin-init-fs.c \ src/ostree/ot-admin-builtin-diff.c \ src/ostree/ot-admin-builtin-deploy.c \ + src/ostree/ot-admin-builtin-finalize-staged.c \ src/ostree/ot-admin-builtin-undeploy.c \ src/ostree/ot-admin-builtin-instutil.c \ src/ostree/ot-admin-builtin-cleanup.c \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 55f2e7a9..94206e03 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -170,6 +170,7 @@ ostree_deployment_get_origin ostree_deployment_get_origin_relpath ostree_deployment_get_unlocked ostree_deployment_is_pinned +ostree_deployment_is_staged ostree_deployment_set_index ostree_deployment_set_bootserial ostree_deployment_set_bootconfig @@ -506,6 +507,7 @@ ostree_sysroot_cleanup ostree_sysroot_prepare_cleanup ostree_sysroot_repo ostree_sysroot_get_repo +ostree_sysroot_get_staged_deployment ostree_sysroot_init_osname ostree_sysroot_deployment_set_kargs ostree_sysroot_deployment_set_mutable @@ -514,6 +516,7 @@ ostree_sysroot_deployment_set_pinned ostree_sysroot_write_deployments ostree_sysroot_write_deployments_with_options ostree_sysroot_write_origin_file +ostree_sysroot_stage_tree ostree_sysroot_deploy_tree ostree_sysroot_get_merge_deployment ostree_sysroot_query_deployments_for diff --git a/src/boot/ostree-finalize-staged.service b/src/boot/ostree-finalize-staged.service new file mode 100644 index 00000000..570138cd --- /dev/null +++ b/src/boot/ostree-finalize-staged.service @@ -0,0 +1,36 @@ +# Copyright (C) 2018 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# For some implementation discussion, see: +# https://lists.freedesktop.org/archives/systemd-devel/2018-March/040557.html +[Unit] +Description=OSTree Finalize Staged Deployment +ConditionPathExists=/run/ostree-booted +DefaultDependencies=no + +RequiresMountsFor=/sysroot +After=basic.target +Before=multi-user.target final.target +Conflicts=final.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStop=/usr/bin/ostree admin finalize-staged + +[Install] +WantedBy=multi-user.target diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 3377ae12..07e11cb6 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,9 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.5 { + ostree_sysroot_stage_tree; + ostree_sysroot_get_staged_deployment; + ostree_deployment_is_staged; } LIBOSTREE_2018.3; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index 49d3f5e5..de82521c 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -26,7 +26,7 @@ #include "ostree-core-private.h" #include "ostree-repo-pull-private.h" #include "ostree-repo-static-delta-private.h" -#include "ostree-sysroot.h" +#include "ostree-sysroot-private.h" #include "ostree-bootloader-grub2.h" #include "otutil.h" @@ -52,7 +52,8 @@ ostree_cmd__private__ (void) _ostree_repo_static_delta_dump, _ostree_repo_static_delta_query_exists, _ostree_repo_static_delta_delete, - _ostree_repo_verify_bindings + _ostree_repo_verify_bindings, + _ostree_sysroot_finalize_staged, }; return &table; diff --git a/src/libostree/ostree-cmdprivate.h b/src/libostree/ostree-cmdprivate.h index 1ac5a1c8..592157bf 100644 --- a/src/libostree/ostree-cmdprivate.h +++ b/src/libostree/ostree-cmdprivate.h @@ -34,6 +34,7 @@ typedef struct { gboolean (* ostree_static_delta_query_exists) (OstreeRepo *repo, const char *delta_id, gboolean *out_exists, GCancellable *cancellable, GError **error); gboolean (* ostree_static_delta_delete) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); gboolean (* ostree_repo_verify_bindings) (const char *collection_id, const char *ref_name, GVariant *commit, GError **error); + gboolean (* ostree_finalize_staged) (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error); } OstreeCmdPrivateVTable; /* Note this not really "public", we just export the symbol, but not the header */ diff --git a/src/libostree/ostree-deployment-private.h b/src/libostree/ostree-deployment-private.h index 114e2f63..ad77317d 100644 --- a/src/libostree/ostree-deployment-private.h +++ b/src/libostree/ostree-deployment-private.h @@ -36,6 +36,7 @@ G_BEGIN_DECLS * @bootconfig: Bootloader configuration * @origin: How to construct an upgraded version of this tree * @unlocked: The unlocked state + * @staged: TRUE iff this deployment is staged */ struct _OstreeDeployment { @@ -50,6 +51,7 @@ struct _OstreeDeployment OstreeBootconfigParser *bootconfig; GKeyFile *origin; OstreeDeploymentUnlockedState unlocked; + gboolean staged; }; void _ostree_deployment_set_bootcsum (OstreeDeployment *self, const char *bootcsum); diff --git a/src/libostree/ostree-deployment.c b/src/libostree/ostree-deployment.c index 75a5bd1d..820c2632 100644 --- a/src/libostree/ostree-deployment.c +++ b/src/libostree/ostree-deployment.c @@ -339,3 +339,16 @@ ostree_deployment_is_pinned (OstreeDeployment *self) return FALSE; return g_key_file_get_boolean (self->origin, OSTREE_ORIGIN_TRANSIENT_GROUP, "pinned", NULL); } + +/** + * ostree_deployment_is_staged: + * @self: Deployment + * + * Returns: `TRUE` if deployment should be "finalized" at shutdown time + * Since: 2018.3 + */ +gboolean +ostree_deployment_is_staged (OstreeDeployment *self) +{ + return self->staged; +} diff --git a/src/libostree/ostree-deployment.h b/src/libostree/ostree-deployment.h index 612222a2..756e39d2 100644 --- a/src/libostree/ostree-deployment.h +++ b/src/libostree/ostree-deployment.h @@ -73,7 +73,8 @@ OstreeBootconfigParser *ostree_deployment_get_bootconfig (OstreeDeployment *self _OSTREE_PUBLIC GKeyFile *ostree_deployment_get_origin (OstreeDeployment *self); - +_OSTREE_PUBLIC +gboolean ostree_deployment_is_staged (OstreeDeployment *self); _OSTREE_PUBLIC gboolean ostree_deployment_is_pinned (OstreeDeployment *self); diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 1d46222b..3698767f 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -308,6 +308,15 @@ cleanup_old_deployments (OstreeSysroot *self, g_hash_table_replace (active_boot_checksums, bootcsum, bootcsum); } + /* And also the staged deployment, if any */ + if (self->staged_deployment) + { + char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, self->staged_deployment); + g_hash_table_replace (active_deployment_dirs, deployment_path, deployment_path); + char *bootcsum = g_strdup (ostree_deployment_get_bootcsum (self->staged_deployment)); + g_hash_table_replace (active_boot_checksums, bootcsum, bootcsum); + } + /* Find all deployment directories, both active and inactive */ g_autoptr(GPtrArray) all_deployment_dirs = NULL; if (!list_all_deployment_directories (self, &all_deployment_dirs, diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 927809e9..b593ce38 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -684,10 +684,15 @@ selinux_relabel_dir (OstreeSysroot *sysroot, static gboolean selinux_relabel_var_if_needed (OstreeSysroot *sysroot, OstreeSePolicy *sepolicy, - int os_deploy_dfd, + OstreeDeployment *deployment, GCancellable *cancellable, GError **error) { + const char *osdeploypath = glnx_strjoina ("ostree/deploy/", ostree_deployment_get_osname (deployment)); + glnx_autofd int os_deploy_dfd = -1; + if (!glnx_opendirat (sysroot->sysroot_fd, osdeploypath, TRUE, &os_deploy_dfd, error)) + return FALSE; + /* This is a bit of a hack; we should change the code at some * point in the distant future to only create (and label) /var * when doing a deployment. @@ -743,12 +748,10 @@ prepare_deployment_etc (OstreeSysroot *sysroot, OstreeRepo *repo, OstreeDeployment *deployment, int deployment_dfd, - OstreeSePolicy **out_sepolicy, GCancellable *cancellable, GError **error) { GLNX_AUTO_PREFIX_ERROR ("Preparing /etc", error); - g_autoptr(OstreeSePolicy) sepolicy = NULL; struct stat stbuf; if (!glnx_fstatat_allow_noent (deployment_dfd, "etc", &stbuf, AT_SYMLINK_NOFOLLOW, error)) @@ -781,7 +784,7 @@ prepare_deployment_etc (OstreeSysroot *sysroot, /* Here, we initialize SELinux policy from the /usr/etc inside * the root - this is before we've finalized the configuration * merge into /etc. */ - sepolicy = ostree_sepolicy_new_at (deployment_dfd, cancellable, error); + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new_at (deployment_dfd, cancellable, error); if (!sepolicy) return FALSE; if (ostree_sepolicy_get_name (sepolicy) != NULL) @@ -796,8 +799,6 @@ prepare_deployment_etc (OstreeSysroot *sysroot, } - if (out_sepolicy) - *out_sepolicy = g_steal_pointer (&sepolicy); return TRUE; } @@ -831,7 +832,6 @@ write_origin_file_internal (OstreeSysroot *sysroot, ostree_deployment_get_csum (deployment), ostree_deployment_get_deployserial (deployment)); - gsize len; g_autofree char *contents = g_key_file_to_data (origin, &len, error); if (!contents) @@ -2324,46 +2324,47 @@ allocate_deployserial (OstreeSysroot *self, return TRUE; } -/** - * ostree_sysroot_deploy_tree: - * @self: Sysroot - * @osname: (allow-none): osname to use for merge deployment - * @revision: Checksum to add - * @origin: (allow-none): Origin to use for upgrades - * @provided_merge_deployment: (allow-none): Use this deployment for merge path - * @override_kernel_argv: (allow-none) (array zero-terminated=1) (element-type utf8): Use these as kernel arguments; if %NULL, inherit options from provided_merge_deployment - * @out_new_deployment: (out): The new deployment path - * @cancellable: Cancellable - * @error: Error - * - * Check out deployment tree with revision @revision, performing a 3 - * way merge with @provided_merge_deployment for configuration. +void +_ostree_deployment_set_bootconfig_from_kargs (OstreeDeployment *deployment, + char **override_kernel_argv) +{ + /* Create an empty boot configuration; we will merge things into + * it as we go. + */ + g_autoptr(OstreeBootconfigParser) bootconfig = ostree_bootconfig_parser_new (); + ostree_deployment_set_bootconfig (deployment, bootconfig); + + /* After this, install_deployment_kernel() will set the other boot + * options and write it out to disk. + */ + if (override_kernel_argv) + { + g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_new (); + _ostree_kernel_args_append_argv (kargs, override_kernel_argv); + g_autofree char *new_options = _ostree_kernel_args_to_string (kargs); + ostree_bootconfig_parser_set (bootconfig, "options", new_options); + } +} + +/* The first part of writing a deployment. This primarily means doing the + * hardlink farm checkout, but we also compute some initial state. */ -gboolean -ostree_sysroot_deploy_tree (OstreeSysroot *self, - const char *osname, - const char *revision, - GKeyFile *origin, - OstreeDeployment *provided_merge_deployment, - char **override_kernel_argv, - OstreeDeployment **out_new_deployment, - GCancellable *cancellable, - GError **error) +static gboolean +sysroot_initialize_deployment (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + char **override_kernel_argv, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error) { g_return_val_if_fail (osname != NULL || self->booted_deployment != NULL, FALSE); if (osname == NULL) osname = ostree_deployment_get_osname (self->booted_deployment); - const char *osdeploypath = glnx_strjoina ("ostree/deploy/", osname); - glnx_autofd int os_deploy_dfd = -1; - if (!glnx_opendirat (self->sysroot_fd, osdeploypath, TRUE, &os_deploy_dfd, error)) - return FALSE; - OstreeRepo *repo = ostree_sysroot_repo (self); - g_autoptr(OstreeDeployment) merge_deployment = NULL; - if (provided_merge_deployment != NULL) - merge_deployment = g_object_ref (provided_merge_deployment); gint new_deployserial; if (!allocate_deployserial (self, osname, revision, &new_deployserial, @@ -2388,66 +2389,328 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, return FALSE; _ostree_deployment_set_bootcsum (new_deployment, kernel_layout->bootcsum); + _ostree_deployment_set_bootconfig_from_kargs (new_deployment, override_kernel_argv); - /* Initial empty boot configuration. */ - g_autoptr(OstreeBootconfigParser) bootconfig = ostree_bootconfig_parser_new (); - ostree_deployment_set_bootconfig (new_deployment, bootconfig); + if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, + cancellable, error)) + return FALSE; - /* Handle kernel arguments. After this, install_deployment_kernel() will set - * the other boot options and write it out to disk. - */ - if (override_kernel_argv) + ot_transfer_out_value (out_new_deployment, &new_deployment); + return TRUE; +} + +static gboolean +sysroot_finalize_deployment (OstreeSysroot *self, + OstreeDeployment *deployment, + char **override_kernel_argv, + OstreeDeployment *merge_deployment, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); + glnx_autofd int deployment_dfd = -1; + if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE, &deployment_dfd, error)) + return FALSE; + + /* Only use the merge if we didn't get an override */ + if (!override_kernel_argv && merge_deployment) { - /* We have an override set, use it */ - g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_new (); - _ostree_kernel_args_append_argv (kargs, override_kernel_argv); - g_autofree char *new_options = _ostree_kernel_args_to_string (kargs); - ostree_bootconfig_parser_set (bootconfig, "options", new_options); - } - else if (provided_merge_deployment) - { - /* Use the merge options by default */ - OstreeBootconfigParser *merge_bootconfig = ostree_deployment_get_bootconfig (provided_merge_deployment); + /* Override the bootloader arguments */ + OstreeBootconfigParser *merge_bootconfig = ostree_deployment_get_bootconfig (merge_deployment); if (merge_bootconfig) { const char *opts = ostree_bootconfig_parser_get (merge_bootconfig, "options"); - ostree_bootconfig_parser_set (bootconfig, "options", opts); + ostree_bootconfig_parser_set (ostree_deployment_get_bootconfig (deployment), "options", opts); } - } - g_autoptr(OstreeSePolicy) sepolicy = NULL; - if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, - &sepolicy, cancellable, error)) - return FALSE; + } if (merge_deployment) { - if (!merge_configuration_from (self, merge_deployment, - new_deployment, deployment_dfd, + /* And do the /etc merge */ + if (!merge_configuration_from (self, merge_deployment, deployment, deployment_dfd, cancellable, error)) return FALSE; } - if (!selinux_relabel_var_if_needed (self, sepolicy, os_deploy_dfd, - cancellable, error)) + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new_at (deployment_dfd, cancellable, error); + if (!sepolicy) return FALSE; - if (!(self->debug_flags & OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS)) - { - if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE, - cancellable, error)) - return FALSE; - } + if (!selinux_relabel_var_if_needed (self, sepolicy, deployment, cancellable, error)) + return FALSE; - /* Don't fsync here, as we assume that's all done in - * ostree_sysroot_write_deployments(). + /* Rewrite the origin using the final merged selinux config, just to be + * conservative about getting the right labels. */ - if (!write_origin_file_internal (self, sepolicy, new_deployment, NULL, + if (!write_origin_file_internal (self, sepolicy, deployment, + ostree_deployment_get_origin (deployment), GLNX_FILE_REPLACE_NODATASYNC, cancellable, error)) return FALSE; - ot_transfer_out_value (out_new_deployment, &new_deployment); + /* Seal it */ + if (!(self->debug_flags & OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS)) + { + if (!ostree_sysroot_deployment_set_mutable (self, deployment, FALSE, + cancellable, error)) + return FALSE; + } + + return TRUE; +} + +/** + * ostree_sysroot_deploy_tree: + * @self: Sysroot + * @osname: (allow-none): osname to use for merge deployment + * @revision: Checksum to add + * @origin: (allow-none): Origin to use for upgrades + * @provided_merge_deployment: (allow-none): Use this deployment for merge path + * @override_kernel_argv: (allow-none) (array zero-terminated=1) (element-type utf8): Use these as kernel arguments; if %NULL, inherit options from provided_merge_deployment + * @out_new_deployment: (out): The new deployment path + * @cancellable: Cancellable + * @error: Error + * + * Check out deployment tree with revision @revision, performing a 3 + * way merge with @provided_merge_deployment for configuration. + * + * While this API is not deprecated, you most likely want to use the + * ostree_sysroot_stage_tree() API. + */ +gboolean +ostree_sysroot_deploy_tree (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *provided_merge_deployment, + char **override_kernel_argv, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(OstreeDeployment) deployment = NULL; + if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv, + &deployment, cancellable, error)) + return FALSE; + + if (!sysroot_finalize_deployment (self, deployment, override_kernel_argv, + provided_merge_deployment, + cancellable, error)) + return FALSE; + + *out_new_deployment = g_steal_pointer (&deployment); + return TRUE; +} + +/* Serialize information about a deployment to a variant, used by the staging + * code. + */ +static GVariant * +serialize_deployment_to_variant (OstreeDeployment *deployment) +{ + g_auto(GVariantBuilder) builder = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_builder_init (&builder, (GVariantType*)"a{sv}"); + g_autofree char *name = + g_strdup_printf ("%s.%d", ostree_deployment_get_csum (deployment), + ostree_deployment_get_deployserial (deployment)); + g_variant_builder_add (&builder, "{sv}", "name", + g_variant_new_string (name)); + g_variant_builder_add (&builder, "{sv}", "osname", + g_variant_new_string (ostree_deployment_get_osname (deployment))); + g_variant_builder_add (&builder, "{sv}", "bootcsum", + g_variant_new_string (ostree_deployment_get_bootcsum (deployment))); + + return g_variant_builder_end (&builder); +} + +static gboolean +require_str_key (GVariantDict *dict, + const char *name, + const char **ret, + GError **error) +{ + if (!g_variant_dict_lookup (dict, name, "&s", ret)) + return glnx_throw (error, "Missing key: %s", name); + return TRUE; +} + +/* Reverse of the above; convert a variant to a deployment. Note that the + * deployment may not actually be present; this should be verified by + * higher level code. + */ +OstreeDeployment * +_ostree_sysroot_deserialize_deployment_from_variant (GVariant *v, + GError **error) +{ + g_autoptr(GVariantDict) dict = g_variant_dict_new (v); + const char *name = NULL; + if (!require_str_key (dict, "name", &name, error)) + return FALSE; + const char *bootcsum = NULL; + if (!require_str_key (dict, "bootcsum", &bootcsum, error)) + return FALSE; + const char *osname = NULL; + if (!require_str_key (dict, "osname", &osname, error)) + return FALSE; + g_autofree char *checksum = NULL; + gint deployserial; + if (!_ostree_sysroot_parse_deploy_path_name (name, &checksum, &deployserial, error)) + return NULL; + return ostree_deployment_new (-1, osname, checksum, deployserial, + bootcsum, -1); +} + + +/** + * ostree_sysroot_stage_tree: + * @self: Sysroot + * @osname: (allow-none): osname to use for merge deployment + * @revision: Checksum to add + * @origin: (allow-none): Origin to use for upgrades + * @merge_deployment: (allow-none): Use this deployment for merge path + * @override_kernel_argv: (allow-none) (array zero-terminated=1) (element-type utf8): Use these as kernel arguments; if %NULL, inherit options from provided_merge_deployment + * @out_new_deployment: (out): The new deployment path + * @cancellable: Cancellable + * @error: Error + * + * Like ostree_sysroot_deploy_tree(), but "finalization" only occurs at OS + * shutdown time. + */ +gboolean +ostree_sysroot_stage_tree (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *merge_deployment, + char **override_kernel_argv, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error) +{ + /* This is a bit of a hack. When adding a new service we have to end up getting + * into the presets for downstream distros; see e.g. https://src.fedoraproject.org/rpms/ostree/pull-request/7 + * + * Then again, it's perhaps a bit nicer to only start the service on-demand anyways. + */ + const char *const systemctl_argv[] = {"systemctl", "start", "ostree-finalize-staged.service", NULL}; + int estatus; + if (!g_spawn_sync (NULL, (char**)systemctl_argv, NULL, G_SPAWN_SEARCH_PATH, + NULL, NULL, NULL, NULL, &estatus, error)) + return FALSE; + if (!g_spawn_check_exit_status (estatus, error)) + return FALSE; + + g_autoptr(OstreeDeployment) deployment = NULL; + if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv, + &deployment, cancellable, error)) + return FALSE; + + /* Write out the origin file using the sepolicy from the non-merged root for + * now (i.e. using /usr/etc policy, not /etc); in practice we don't really + * expect people to customize the label for it. + */ + { g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); + glnx_autofd int deployment_dfd = -1; + if (!glnx_opendirat (self->sysroot_fd, deployment_path, FALSE, + &deployment_dfd, error)) + return FALSE; + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new_at (deployment_dfd, cancellable, error); + if (!sepolicy) + return FALSE; + if (!write_origin_file_internal (self, sepolicy, deployment, + ostree_deployment_get_origin (deployment), + GLNX_FILE_REPLACE_NODATASYNC, + cancellable, error)) + return FALSE; + } + + /* After here we defer action until shutdown. The remaining arguments (merge + * deployment, kargs) are serialized to a state file in /run. + */ + + /* "target" is the staged deployment */ + g_autoptr(GVariantBuilder) builder = g_variant_builder_new ((GVariantType*)"a{sv}"); + g_variant_builder_add (builder, "{sv}", "target", + serialize_deployment_to_variant (deployment)); + + if (merge_deployment) + g_variant_builder_add (builder, "{sv}", "merge-deployment", + serialize_deployment_to_variant (merge_deployment)); + + if (override_kernel_argv) + g_variant_builder_add (builder, "{sv}", "kargs", + g_variant_new_strv ((const char *const*)override_kernel_argv, -1)); + + const char *parent = dirname (strdupa (_OSTREE_SYSROOT_RUNSTATE_STAGED)); + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, parent, 0755, cancellable, error)) + return FALSE; + + g_autoptr(GVariant) state = g_variant_ref_sink (g_variant_builder_end (builder)); + if (!glnx_file_replace_contents_at (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED, + g_variant_get_data (state), g_variant_get_size (state), + GLNX_FILE_REPLACE_NODATASYNC, + cancellable, error)) + return FALSE; + + /* If we have a previous one, clean it up */ + if (self->staged_deployment) + { + if (!_ostree_sysroot_rmrf_deployment (self, self->staged_deployment, cancellable, error)) + return FALSE; + } + + if (!_ostree_sysroot_reload_staged (self, error)) + return FALSE; + + return TRUE; +} + +/* Invoked at shutdown time by ostree-finalize-staged.service */ +gboolean +_ostree_sysroot_finalize_staged (OstreeSysroot *self, + GCancellable *cancellable, + GError **error) +{ + /* It's totally fine if there's no staged deployment; perhaps down the line + * though we could teach the ostree cmdline to tell systemd to activate the + * service when a staged deployment is created. + */ + if (!self->staged_deployment) + return TRUE; + + g_assert (self->staged_deployment_data); + + g_autoptr(OstreeDeployment) merge_deployment = NULL; + g_autoptr(GVariant) merge_deployment_v = NULL; + if (g_variant_lookup (self->staged_deployment_data, "merge-deployment", "@a{sv}", + &merge_deployment_v)) + { + merge_deployment = + _ostree_sysroot_deserialize_deployment_from_variant (merge_deployment_v, error); + if (!merge_deployment) + return FALSE; + } + g_autofree char **kargs = NULL; + g_variant_lookup (self->staged_deployment_data, "kargs", "^a&s", &kargs); + + /* Unlink the staged state now; if we're interrupted in the middle, + * we don't want e.g. deal with the partially written /etc merge. + */ + if (!glnx_unlinkat (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED, 0, error)) + return FALSE; + + if (!sysroot_finalize_deployment (self, self->staged_deployment, NULL, merge_deployment, + cancellable, error)) + return FALSE; + + /* TODO: Proxy across flags too? */ + OstreeSysrootSimpleWriteDeploymentFlags flags = 0; + if (!ostree_sysroot_simple_write_deployment (self, ostree_deployment_get_osname (self->staged_deployment), + self->staged_deployment, merge_deployment, flags, + cancellable, error)) + return FALSE; + return TRUE; } diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 01b370e8..a2f3b869 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -61,6 +61,8 @@ struct OstreeSysroot { int bootversion; int subbootversion; OstreeDeployment *booted_deployment; + OstreeDeployment *staged_deployment; + GVariant *staged_deployment_data; struct timespec loaded_ts; /* Only access through ostree_sysroot_[_get]repo() */ @@ -71,6 +73,7 @@ struct OstreeSysroot { #define OSTREE_SYSROOT_LOCKFILE "ostree/lock" /* We keep some transient state in /run */ +#define _OSTREE_SYSROOT_RUNSTATE_STAGED "/run/ostree/staged-deployment" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_DIR "/run/ostree/deployment-state/" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT "unlocked-development" @@ -105,6 +108,22 @@ _ostree_sysroot_list_deployment_dirs_for_os (int deploydir_dfd, GCancellable *cancellable, GError **error); +void +_ostree_deployment_set_bootconfig_from_kargs (OstreeDeployment *deployment, + char **override_kernel_argv); + +gboolean +_ostree_sysroot_reload_staged (OstreeSysroot *self, GError **error); + +gboolean +_ostree_sysroot_finalize_staged (OstreeSysroot *self, + GCancellable *cancellable, + GError **error); + +OstreeDeployment * +_ostree_sysroot_deserialize_deployment_from_variant (GVariant *v, + GError **error); + char * _ostree_sysroot_get_origin_relpath (GFile *path, guint32 *out_device, @@ -118,6 +137,8 @@ _ostree_sysroot_rmrf_deployment (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error); +char * _ostree_sysroot_get_runstate_path (OstreeDeployment *deployment, const char *key); + char *_ostree_sysroot_join_lines (GPtrArray *lines); gboolean _ostree_sysroot_query_bootloader (OstreeSysroot *sysroot, diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index f77d7703..f4a8eade 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -82,6 +82,8 @@ ostree_sysroot_finalize (GObject *object) g_clear_object (&self->repo); g_clear_pointer (&self->deployments, g_ptr_array_unref); g_clear_object (&self->booted_deployment); + g_clear_object (&self->staged_deployment); + g_clear_pointer (&self->staged_deployment_data, (GDestroyNotify)g_variant_unref); glnx_release_lock_file (&self->lock); @@ -584,14 +586,14 @@ parse_bootlink (const char *bootlink, return TRUE; } -static char * -get_unlocked_development_path (OstreeDeployment *deployment) +char * +_ostree_sysroot_get_runstate_path (OstreeDeployment *deployment, const char *key) { return g_strdup_printf ("%s%s.%d/%s", _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_DIR, ostree_deployment_get_csum (deployment), ostree_deployment_get_deployserial (deployment), - _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT); + key); } static gboolean @@ -636,9 +638,10 @@ parse_deployment (OstreeSysroot *self, return FALSE; /* See if this is the booted deployment */ + const gboolean root_is_ostree_booted = + (self->ostree_booted && self->root_is_sysroot); const gboolean looking_for_booted_deployment = - (self->ostree_booted && self->root_is_sysroot && - !self->booted_deployment); + (root_is_ostree_booted && !self->booted_deployment); gboolean is_booted_deployment = FALSE; if (looking_for_booted_deployment) { @@ -665,7 +668,8 @@ parse_deployment (OstreeSysroot *self, ostree_deployment_set_origin (ret_deployment, origin); ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_NONE; - g_autofree char *unlocked_development_path = get_unlocked_development_path (ret_deployment); + g_autofree char *unlocked_development_path = + _ostree_sysroot_get_runstate_path (ret_deployment, _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT); struct stat stbuf; if (lstat (unlocked_development_path, &stbuf) == 0) ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_DEVELOPMENT; @@ -789,6 +793,60 @@ ensure_repo (OstreeSysroot *self, return TRUE; } +/* Reload the staged deployment from the file in /run */ +gboolean +_ostree_sysroot_reload_staged (OstreeSysroot *self, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR ("Loading staged deployment", error); + const gboolean root_is_ostree_booted = + self->ostree_booted && self->root_is_sysroot; + if (!root_is_ostree_booted) + return TRUE; /* Note early return */ + + g_assert (self->booted_deployment); + + g_clear_object (&self->staged_deployment); + g_clear_pointer (&self->staged_deployment_data, (GDestroyNotify)g_variant_unref); + + /* Read the staged state from disk */ + glnx_autofd int fd = -1; + if (!ot_openat_ignore_enoent (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED, &fd, error)) + return FALSE; + if (fd != -1) + { + g_autoptr(GBytes) contents = ot_fd_readall_or_mmap (fd, 0, error); + if (!contents) + return FALSE; + g_autoptr(GVariant) staged_deployment_data = + g_variant_new_from_bytes ((GVariantType*)"a{sv}", contents, TRUE); + g_autoptr(GVariantDict) staged_deployment_dict = + g_variant_dict_new (staged_deployment_data); + + /* Parse it */ + g_autoptr(GVariant) target = NULL; + g_autofree char **kargs = NULL; + g_variant_dict_lookup (staged_deployment_dict, "target", "@a{sv}", &target); + g_variant_dict_lookup (staged_deployment_dict, "kargs", "^a&s", &kargs); + if (target) + { + self->staged_deployment = + _ostree_sysroot_deserialize_deployment_from_variant (target, error); + if (!self->staged_deployment) + return FALSE; + _ostree_deployment_set_bootconfig_from_kargs (self->staged_deployment, kargs); + self->staged_deployment_data = g_steal_pointer (&staged_deployment_data); + /* We set this flag for ostree_deployment_is_staged() because that API + * doesn't have access to the sysroot, which currently has the + * canonical "staged_deployment" reference. + */ + self->staged_deployment->staged = TRUE; + } + } + + return TRUE; +} + gboolean ostree_sysroot_load_if_changed (OstreeSysroot *self, gboolean *out_changed, @@ -857,6 +915,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, g_clear_pointer (&self->deployments, g_ptr_array_unref); g_clear_object (&self->booted_deployment); + g_clear_object (&self->staged_deployment); self->bootversion = -1; self->subbootversion = -1; @@ -880,17 +939,23 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, } } - if (self->ostree_booted && self->root_is_sysroot - && !self->booted_deployment) + const gboolean root_is_ostree_booted = + self->ostree_booted && self->root_is_sysroot; + if (root_is_ostree_booted && !self->booted_deployment) return glnx_throw (error, "Unexpected state: /run/ostree-booted found and in / sysroot but not in a booted deployment"); + /* Ensure the entires are sorted */ g_ptr_array_sort (deployments, compare_deployments_by_boot_loader_version_reversed); + /* And then set their index variables */ for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; ostree_deployment_set_index (deployment, i); } + if (!_ostree_sysroot_reload_staged (self, error)) + return FALSE; + /* Determine whether we're "physical" or not, the first time we initialize */ if (!self->loaded) { @@ -949,6 +1014,20 @@ ostree_sysroot_get_booted_deployment (OstreeSysroot *self) return self->booted_deployment; } +/** + * ostree_sysroot_get_staged_deployment: + * @self: Sysroot + * + * Returns: (transfer none): The currently staged deployment, or %NULL if none + */ +OstreeDeployment * +ostree_sysroot_get_staged_deployment (OstreeSysroot *self) +{ + g_return_val_if_fail (self->loaded, NULL); + + return self->staged_deployment; +} + /** * ostree_sysroot_get_deployments: * @self: Sysroot @@ -1769,7 +1848,8 @@ ostree_sysroot_deployment_unlock (OstreeSysroot *self, break; case OSTREE_DEPLOYMENT_UNLOCKED_DEVELOPMENT: { - g_autofree char *devpath = get_unlocked_development_path (deployment); + g_autofree char *devpath = + _ostree_sysroot_get_runstate_path (deployment, _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT); g_autofree char *devpath_parent = dirname (g_strdup (devpath)); if (!glnx_shutil_mkdir_p_at (AT_FDCWD, devpath_parent, 0755, cancellable, error)) diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index e4763d37..47cbb022 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -74,6 +74,8 @@ _OSTREE_PUBLIC GPtrArray *ostree_sysroot_get_deployments (OstreeSysroot *self); _OSTREE_PUBLIC OstreeDeployment *ostree_sysroot_get_booted_deployment (OstreeSysroot *self); +_OSTREE_PUBLIC +OstreeDeployment *ostree_sysroot_get_staged_deployment (OstreeSysroot *self); _OSTREE_PUBLIC GFile *ostree_sysroot_get_deployment_directory (OstreeSysroot *self, @@ -174,6 +176,17 @@ gboolean ostree_sysroot_deploy_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sysroot_stage_tree (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *merge_deployment, + char **override_kernel_argv, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_deployment_set_mutable (OstreeSysroot *self, OstreeDeployment *deployment, diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index d9905212..f6c0c161 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -34,6 +34,7 @@ #include static gboolean opt_retain; +static gboolean opt_stage; static gboolean opt_retain_pending; static gboolean opt_retain_rollback; static gboolean opt_not_as_default; @@ -50,6 +51,7 @@ static GOptionEntry options[] = { { "origin-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_origin_path, "Specify origin file", "FILENAME" }, { "no-prune", 0, 0, G_OPTION_ARG_NONE, &opt_no_prune, "Don't prune the repo when done", NULL}, { "retain", 0, 0, G_OPTION_ARG_NONE, &opt_retain, "Do not delete previous deployments", NULL }, + { "stage", 0, 0, G_OPTION_ARG_NONE, &opt_stage, "Complete deployment at OS shutdown", NULL }, { "retain-pending", 0, 0, G_OPTION_ARG_NONE, &opt_retain_pending, "Do not delete pending deployments", NULL }, { "retain-rollback", 0, 0, G_OPTION_ARG_NONE, &opt_retain_rollback, "Do not delete rollback deployments", NULL }, { "not-as-default", 0, 0, G_OPTION_ARG_NONE, &opt_not_as_default, "Append rather than prepend new deployment", NULL }, @@ -157,31 +159,45 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat g_autoptr(OstreeDeployment) new_deployment = NULL; g_auto(GStrv) kargs_strv = _ostree_kernel_args_to_strv (kargs); - if (!ostree_sysroot_deploy_tree (sysroot, opt_osname, revision, origin, merge_deployment, - kargs_strv, &new_deployment, cancellable, error)) - return FALSE; - - OstreeSysrootSimpleWriteDeploymentFlags flags = OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN; - if (opt_retain) - flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN; + if (opt_stage) + { + if (opt_retain_pending || opt_retain_rollback) + return glnx_throw (error, "--stage cannot currently be combined with --retain arguments"); + if (opt_not_as_default) + return glnx_throw (error, "--stage cannot currently be combined with --not-as-default"); + if (!ostree_sysroot_stage_tree (sysroot, opt_osname, revision, origin, merge_deployment, + kargs_strv, &new_deployment, cancellable, error)) + return FALSE; + } else { - if (opt_retain_pending) - flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PENDING; - if (opt_retain_rollback) - flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_ROLLBACK; + if (!ostree_sysroot_deploy_tree (sysroot, opt_osname, revision, origin, merge_deployment, + kargs_strv, &new_deployment, cancellable, error)) + return FALSE; + + OstreeSysrootSimpleWriteDeploymentFlags flags = OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN; + if (opt_retain) + flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN; + else + { + if (opt_retain_pending) + flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_PENDING; + if (opt_retain_rollback) + flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_RETAIN_ROLLBACK; + } + + if (opt_not_as_default) + flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NOT_DEFAULT; + + if (!ostree_sysroot_simple_write_deployment (sysroot, opt_osname, new_deployment, + merge_deployment, flags, cancellable, error)) + return FALSE; } - if (opt_not_as_default) - flags |= OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NOT_DEFAULT; - - if (!ostree_sysroot_simple_write_deployment (sysroot, opt_osname, new_deployment, - merge_deployment, flags, cancellable, error)) - return FALSE; - - /* And finally, cleanup of any leftover data. + /* And finally, cleanup of any leftover data. In stage mode, we + * don't do a full cleanup as we didn't touch the bootloader. */ - if (opt_no_prune) + if (opt_no_prune || opt_stage) { if (!ostree_sysroot_prepare_cleanup (sysroot, cancellable, error)) return FALSE; diff --git a/src/ostree/ot-admin-builtin-finalize-staged.c b/src/ostree/ot-admin-builtin-finalize-staged.c new file mode 100644 index 00000000..6740f82a --- /dev/null +++ b/src/ostree/ot-admin-builtin-finalize-staged.c @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2018 Red Hat, Inc. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "config.h" + +#include + +#include "ot-main.h" +#include "ot-admin-builtins.h" +#include "ot-admin-functions.h" +#include "ostree.h" +#include "otutil.h" + +#include "ostree-cmdprivate.h" +#include "ostree.h" + +/* Called by ostree-finalize-staged.service, and in turn + * invokes a cmdprivate function inside the shared library. + */ +gboolean +ot_admin_builtin_finalize_staged (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + /* Just a sanity check; we shouldn't be called outside of the service though. + */ + struct stat stbuf; + if (fstatat (AT_FDCWD, "/run/ostree-booted", &stbuf, 0) < 0) + return TRUE; + + g_autoptr(GFile) sysroot_file = g_file_new_for_path ("/"); + g_autoptr(OstreeSysroot) sysroot = ostree_sysroot_new (sysroot_file); + + if (!ostree_sysroot_load (sysroot, cancellable, error)) + return FALSE; + if (!ostree_cmd__private__()->ostree_finalize_staged (sysroot, cancellable, error)) + return FALSE; + + return TRUE; +} diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 096155c6..55be6994 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -96,7 +96,9 @@ deployment_print_status (OstreeSysroot *sysroot, GKeyFile *origin = ostree_deployment_get_origin (deployment); const char *deployment_status = ""; - if (is_pending) + if (ostree_deployment_is_staged (deployment)) + deployment_status = " (staged)"; + else if (is_pending) deployment_status = " (pending)"; else if (is_rollback) deployment_status = " (rollback)"; @@ -199,6 +201,16 @@ ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocat } else { + OstreeDeployment *staged = ostree_sysroot_get_staged_deployment (sysroot); + if (staged) + { + if (!deployment_print_status (sysroot, repo, staged, + FALSE, FALSE, FALSE, + cancellable, + error)) + return FALSE; + } + for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; diff --git a/src/ostree/ot-admin-builtins.h b/src/ostree/ot-admin-builtins.h index a81f4d62..d88fc0b9 100644 --- a/src/ostree/ot-admin-builtins.h +++ b/src/ostree/ot-admin-builtins.h @@ -40,6 +40,7 @@ BUILTINPROTO(undeploy); BUILTINPROTO(deploy); BUILTINPROTO(cleanup); BUILTINPROTO(pin); +BUILTINPROTO(finalize_staged); BUILTINPROTO(unlock); BUILTINPROTO(status); BUILTINPROTO(set_origin); diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index 1262c5a5..b26eea81 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -57,6 +57,9 @@ static OstreeCommand admin_subcommands[] = { { "pin", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_pin, "Change the \"pinning\" state of a deployment" }, + { "finalize-staged", OSTREE_BUILTIN_FLAG_NO_REPO | OSTREE_BUILTIN_FLAG_HIDDEN, + ot_admin_builtin_finalize_staged, + "Internal command to run at shutdown time" }, { "status", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_status, "List deployments" }, diff --git a/tests/installed/destructive.yml b/tests/installed/destructive.yml index 9529c7e9..5bd4d7a7 100644 --- a/tests/installed/destructive.yml +++ b/tests/installed/destructive.yml @@ -16,6 +16,10 @@ # Next copy all of the tests/ directory - name: Copy test data synchronize: src=../../ dest=/root/tests/ archive=yes + + # First, the Ansible-based tests + - import_tasks: destructive/staged-deploy.yml + - find: paths: /root/tests/installed/destructive patterns: "itest-*.sh" diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml new file mode 100644 index 00000000..bf504675 --- /dev/null +++ b/tests/installed/destructive/staged-deploy.yml @@ -0,0 +1,24 @@ +# Test the deploy --stage functionality + +- name: Write staged-deploy commit + shell: | + ostree --repo=/ostree/repo commit --parent="${commit}" -b staged-deploy --tree=ref="${commit}" --no-bindings + ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + environment: + commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" +- include_tasks: ../tasks/reboot.yml +- name: Check that deploy-staged service worked + shell: | + # Assert that the previous boot had a journal entry for it + journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' + # And that we have the new kernel argument + grep -q -e 'ostreetest=yes' /proc/cmdline +- name: Rollback + shell: rpm-ostree rollback +- include_tasks: ../tasks/reboot.yml +- shell: | + ostree refs --delete staged-deploy + rpm-ostree cleanup -rp +# And now we shouldn't have the kernel commandline entry +- name: Check we do not have new kernel cmdline entry + shell: grep -qv -e 'ostreetest=yes' /proc/cmdline diff --git a/tests/installed/tasks/reboot.yml b/tests/installed/tasks/reboot.yml new file mode 100644 index 00000000..fd077102 --- /dev/null +++ b/tests/installed/tasks/reboot.yml @@ -0,0 +1,71 @@ +# This file is copied from atomic-host-tests + +# vim: set ft=ansible: +# There is no clean way to restart hosts in ansible. The general issue is that +# the shutdown command may close sshd before ansible has time to "return" from +# the task, even with async & poll. This is due to the fact that asynchronous +# tasks still require a small synchronous bootstrapping script which takes 1 sec +# to complete, during which it is vulnerable to erroring out if sshd dies. +# To mitigate this, we prefix a sleep command before the shutdown so +# ansible has time to move on. For more info on this issue, see: +# https://github.com/ansible/ansible/issues/10616 +# +# The Ansible docs now recommend this combination of tasks to handle reboots +# https://support.ansible.com/hc/en-us/articles/201958037-Reboot-a-server-and-wait-for-it-to-come-back + +# remember the real ansible_host for following local actions +# (otherwise ansible will target the localhost) +- set_fact: + real_ansible_host: "{{ ansible_host }}" + timeout: "{{ cli_reboot_timeout | default('120') }}" + +# Have to account for both because Fedora STR uses the old version of these +# inventory values for some reason. +- when: ansible_port is defined + set_fact: + real_ansible_port: "{{ ansible_port }}" + +- when: ansible_ssh_port is defined + set_fact: + real_ansible_port: "{{ ansible_ssh_port }}" + +- name: Get original bootid + command: cat /proc/sys/kernel/random/boot_id + register: orig_bootid + +- name: restart hosts + when: (not skip_shutdown is defined) or (not skip_shutdown) + shell: sleep 3 && shutdown -r now + async: 1 + poll: 0 + ignore_errors: true + +# NB: The following tasks use local actions, so we need to explicitly ensure +# that they don't use sudo, which may require a password, and is not necessary +# anyway. + +- name: wait for hosts to come back up + local_action: + wait_for host={{ real_ansible_host }} + port={{ real_ansible_port | default('22') }} + state=started + delay=30 + timeout={{ timeout }} + search_regex="OpenSSH" + become: false + +# I'm not sure the retries are even necessary, but I'm keeping them in +- name: Wait until bootid changes + command: cat /proc/sys/kernel/random/boot_id + register: new_bootid + until: new_bootid.stdout != orig_bootid.stdout + retries: 6 + delay: 10 + +# provide an empty iterator when a list is not provided +# http://docs.ansible.com/ansible/playbooks_conditionals.html#loops-and-conditionals +- name: check services have started + service: + name: "{{ item }}" + state: started + with_items: "{{ wait_for_services|default([]) }}" From 56de6317212c15fed003e4ac2b52f5c4eb6d498c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 Apr 2018 12:43:39 -0400 Subject: [PATCH 27/80] deploy: Clean up bootserial assignment function The reason we were returning a hashtable is a bit lost to history, there's no reason to do so now anyways. Also port to declare-and-initialize style and add more comments. Closes: #1538 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index b593ce38..d3540c8d 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1874,26 +1874,26 @@ swap_bootloader (OstreeSysroot *sysroot, return TRUE; } -static GHashTable * +/* Deployments may share boot checksums; the bootserial indexes them + * per-bootchecksum. It's used by the symbolic links after the bootloader. + */ +static void assign_bootserials (GPtrArray *deployments) { - guint i; - GHashTable *ret = + g_autoptr(GHashTable) serials = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); - for (i = 0; i < deployments->len; i++) + for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; const char *bootcsum = ostree_deployment_get_bootcsum (deployment); - guint count; - - count = GPOINTER_TO_UINT (g_hash_table_lookup (ret, bootcsum)); - g_hash_table_replace (ret, (char*) bootcsum, + /* Note that not-found maps to NULL which converts to zero */ + guint count = GPOINTER_TO_UINT (g_hash_table_lookup (serials, bootcsum)); + g_hash_table_replace (serials, (char*) bootcsum, GUINT_TO_POINTER (count + 1)); ostree_deployment_set_bootserial (deployment, count); } - return ret; } /* OSTree implements a special optimization where we want to avoid touching @@ -2146,7 +2146,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, /* Assign a bootserial to each new deployment. */ - g_hash_table_unref (assign_bootserials (new_deployments)); + assign_bootserials (new_deployments); /* Determine whether or not we need to touch the bootloader * configuration. If we have an equal number of deployments with From efdaf1495b5f8e8953f0d5174a478028639b9dac Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 Apr 2018 12:53:17 -0400 Subject: [PATCH 28/80] deploy: Silently do nothing if passed same set of deployments Prep for handling staged deployments better; if we're not passed the staged one back, then we just want to delete it but not touch the bootloader config. Closes: #1538 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index d3540c8d..e7dc25d0 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2158,14 +2158,26 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, requires_new_bootversion = TRUE; else { + gboolean is_noop = TRUE; for (guint i = 0; i < new_deployments->len; i++) { - if (!deployment_bootconfigs_equal (new_deployments->pdata[i], - self->deployments->pdata[i])) + OstreeDeployment *cur_deploy = self->deployments->pdata[i]; + OstreeDeployment *new_deploy = new_deployments->pdata[i]; + if (!deployment_bootconfigs_equal (cur_deploy, new_deploy)) { requires_new_bootversion = TRUE; + is_noop = FALSE; break; } + if (cur_deploy != new_deploy) + is_noop = FALSE; + } + + /* Silently do nothing if we're passed the same set of deployments */ + if (is_noop) + { + g_assert (!requires_new_bootversion); + return TRUE; } } From a2dc73198203ca61be997ba67d8edecce3520bba Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 13 Apr 2018 08:56:28 -0400 Subject: [PATCH 29/80] sysroot: Clean up origin loading function In prep for staging work, where we'll need to load the origin for the staged deployment too. The function was previously trying to avoid operating on an instantiated deployment, but the data we need is in the deployment object at that point. Closes: #1538 Approved by: jlebon --- src/libostree/ostree-sysroot.c | 42 +++++++++++++++------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index f4a8eade..51d51340 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -527,32 +527,30 @@ read_current_bootversion (OstreeSysroot *self, } static gboolean -parse_origin (OstreeSysroot *self, - int deployment_dfd, - const char *deployment_name, - GKeyFile **out_origin, - GCancellable *cancellable, - GError **error) +load_origin (OstreeSysroot *self, + OstreeDeployment *deployment, + GCancellable *cancellable, + GError **error) { - g_autofree char *origin_path = g_strconcat ("../", deployment_name, ".origin", NULL); - g_autoptr(GKeyFile) ret_origin = g_key_file_new (); + g_autofree char *origin_path = ostree_deployment_get_origin_relpath (deployment); - struct stat stbuf; - if (!glnx_fstatat_allow_noent (deployment_dfd, origin_path, &stbuf, 0, error)) + glnx_autofd int fd = -1; + if (!ot_openat_ignore_enoent (self->sysroot_fd, origin_path, &fd, error)) return FALSE; - if (errno == 0) + if (fd >= 0) { g_autofree char *origin_contents = - glnx_file_get_contents_utf8_at (deployment_dfd, origin_path, - NULL, cancellable, error); + glnx_fd_readall_utf8 (fd, NULL, cancellable, error); if (!origin_contents) return FALSE; - if (!g_key_file_load_from_data (ret_origin, origin_contents, -1, 0, error)) + g_autoptr(GKeyFile) origin = g_key_file_new (); + if (!g_key_file_load_from_data (origin, origin_contents, -1, 0, error)) return glnx_prefix_error (error, "Parsing %s", origin_path); + + ostree_deployment_set_origin (deployment, origin); } - ot_transfer_out_value(out_origin, &ret_origin); return TRUE; } @@ -656,16 +654,11 @@ parse_deployment (OstreeSysroot *self, stbuf.st_ino == self->root_inode); } - g_autoptr(GKeyFile) origin = NULL; - if (!parse_origin (self, deployment_dfd, deploy_basename, &origin, - cancellable, error)) - return FALSE; - g_autoptr(OstreeDeployment) ret_deployment = ostree_deployment_new (-1, osname, treecsum, deployserial, bootcsum, treebootserial); - if (origin) - ostree_deployment_set_origin (ret_deployment, origin); + if (!load_origin (self, ret_deployment, cancellable, error)) + return FALSE; ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_NONE; g_autofree char *unlocked_development_path = @@ -675,8 +668,9 @@ parse_deployment (OstreeSysroot *self, ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_DEVELOPMENT; else { - g_autofree char *existing_unlocked_state = - g_key_file_get_string (origin, "origin", "unlocked", NULL); + GKeyFile *origin = ostree_deployment_get_origin (ret_deployment); + g_autofree char *existing_unlocked_state = origin ? + g_key_file_get_string (origin, "origin", "unlocked", NULL) : NULL; if (g_strcmp0 (existing_unlocked_state, "hotfix") == 0) { From fea9277020e040a0be6d0c626262a2a2ff36a318 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 3 Apr 2018 16:22:31 -0400 Subject: [PATCH 30/80] switchroot: Don't log if running as pid1, minor code style cleanups If we're running as pid1, avoid printing anything in the normal success paths as we don't want to affect the physical console by default; the device may be using a splash screen, etc. Also cleanup the code a bit to use a single variable `running_as_pid1`, declare-and-initialize, use the `bool` type, etc. Closes: #1531 Approved by: jlebon --- src/switchroot/ostree-prepare-root.c | 36 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index a5c3c785..5ed9b60f 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,9 @@ #include "ostree-mount-util.h" +/* Initialized early in main */ +static bool running_as_pid1; + static char* resolve_deploy_path (const char * root_mountpoint) { @@ -60,7 +64,6 @@ resolve_deploy_path (const char * root_mountpoint) errx (EXIT_FAILURE, "No OSTree target; expected ostree=/ostree/boot.N/..."); snprintf (destpath, sizeof(destpath), "%s/%s", root_mountpoint, ostree_target); - printf ("Examining %s\n", destpath); if (lstat (destpath, &stbuf) < 0) err (EXIT_FAILURE, "Couldn't find specified OSTree root '%s'", destpath); if (!S_ISLNK (stbuf.st_mode)) @@ -68,7 +71,9 @@ resolve_deploy_path (const char * root_mountpoint) deploy_path = realpath (destpath, NULL); if (deploy_path == NULL) err (EXIT_FAILURE, "realpath(%s) failed", destpath); - printf ("Resolved OSTree target to: %s\n", deploy_path); + /* Quiet logs if there's no journal */ + if (!running_as_pid1) + printf ("Resolved OSTree target to: %s\n", deploy_path); return deploy_path; } @@ -81,13 +86,18 @@ pivot_root(const char * new_root, const char * put_old) int main(int argc, char *argv[]) { - const char *root_mountpoint = NULL, *root_arg = NULL; - char *deploy_path = NULL; - char srcpath[PATH_MAX]; - struct stat stbuf; - int we_mounted_proc = 0; + /* If we're pid 1, that means there's no initramfs; in this situation + * various defaults change: + * + * - Assume that the target root is / + * - Quiet logging as there's no journal + * etc. + */ + running_as_pid1 = (getpid () == 1); - if (getpid() == 1) + const char *root_arg = NULL; + bool we_mounted_proc = false; + if (running_as_pid1) { root_arg = "/"; } @@ -98,6 +108,7 @@ main(int argc, char *argv[]) root_arg = argv[1]; } + struct stat stbuf; if (stat ("/proc/cmdline", &stbuf) < 0) { if (errno != ENOENT) @@ -109,10 +120,10 @@ main(int argc, char *argv[]) we_mounted_proc = 1; } - root_mountpoint = realpath (root_arg, NULL); + const char *root_mountpoint = realpath (root_arg, NULL); if (root_mountpoint == NULL) err (EXIT_FAILURE, "realpath(\"%s\")", root_arg); - deploy_path = resolve_deploy_path (root_mountpoint); + char *deploy_path = resolve_deploy_path (root_mountpoint); if (we_mounted_proc) { @@ -147,6 +158,7 @@ main(int argc, char *argv[]) err (EXIT_FAILURE, "failed to bind mount ../../var to var"); #endif + char srcpath[PATH_MAX]; /* If /boot is on the same partition, use a bind mount to make it visible * at /boot inside the deployment. */ snprintf (srcpath, sizeof(srcpath), "%s/boot/loader", root_mountpoint); @@ -193,7 +205,7 @@ main(int argc, char *argv[]) * not pid 1. Otherwise it's handled later via ostree-remount.service. * https://mail.gnome.org/archives/ostree-list/2018-March/msg00012.html */ - if (getpid () != 1) + if (!running_as_pid1) touch_run_ostree (); if (strcmp(root_mountpoint, "/") == 0) @@ -246,7 +258,7 @@ main(int argc, char *argv[]) if (mount ("none", "sysroot", NULL, MS_PRIVATE, NULL) < 0) err (EXIT_FAILURE, "remounting 'sysroot' private"); - if (getpid() == 1) + if (running_as_pid1) { execl ("/sbin/init", "/sbin/init", NULL); err (EXIT_FAILURE, "failed to exec init inside ostree"); From cdaf7cd8383af6f59a70a74d7c1467d8f7b536be Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 29 Mar 2018 17:10:41 +0200 Subject: [PATCH 31/80] commit, payload-reflink: do not write to the parent repo reintroduce the feature that was reverted with commit: 28c7bc6d0e153a0b07bdb82d25473a490765067f Differently than the original implementation, now we don't attempt any test for reflinks support on the parent repository, since the test requires write access to the repository. Additionally, also check that the two repositories are on the same device before attempting any reflink. Signed-off-by: Giuseppe Scrivano Closes: #1525 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 51 +++++++----- .../nondestructive/itest-payload-link.sh | 83 +++++++++++++++---- 2 files changed, 98 insertions(+), 36 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 5fd97902..c171b3da 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -598,27 +598,26 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, } static gboolean -_check_support_reflink (OstreeRepo *self, gboolean *supported, GError **error) +_check_support_reflink (OstreeRepo *dest, gboolean *supported, GError **error) { - /* We have not checked yet if the file system supports reflinks, do it here */ - if (g_atomic_int_get (&self->fs_support_reflink) == 0) + /* We have not checked yet if the destination file system supports reflinks, do it here */ + if (g_atomic_int_get (&dest->fs_support_reflink) == 0) { - g_auto(GLnxTmpfile) src_tmpf = { 0, }; + glnx_autofd int src_fd = -1; g_auto(GLnxTmpfile) dest_tmpf = { 0, }; - if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_RDWR|O_CLOEXEC, - &src_tmpf, error)) + if (!glnx_openat_rdonly (dest->repo_dir_fd, "config", TRUE, &src_fd, error)) return FALSE; - if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, + if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (dest), ".", O_WRONLY|O_CLOEXEC, &dest_tmpf, error)) return FALSE; - if (ioctl (dest_tmpf.fd, FICLONE, src_tmpf.fd) == 0) - g_atomic_int_set (&self->fs_support_reflink, 1); + if (ioctl (dest_tmpf.fd, FICLONE, src_fd) == 0) + g_atomic_int_set (&dest->fs_support_reflink, 1); else if (errno == EOPNOTSUPP) /* Ignore other kind of errors as they might be temporary failures */ - g_atomic_int_set (&self->fs_support_reflink, -1); + g_atomic_int_set (&dest->fs_support_reflink, -1); } - *supported = g_atomic_int_get (&self->fs_support_reflink) >= 0; + *supported = g_atomic_int_get (&dest->fs_support_reflink) >= 0; return TRUE; } @@ -631,6 +630,7 @@ _create_payload_link (OstreeRepo *self, GError **error) { gboolean reflinks_supported = FALSE; + if (!_check_support_reflink (self, &reflinks_supported, error)) return FALSE; @@ -664,8 +664,8 @@ _create_payload_link (OstreeRepo *self, } static gboolean -_import_payload_link (OstreeRepo *self, - OstreeRepo *source, +_import_payload_link (OstreeRepo *dest_repo, + OstreeRepo *src_repo, const char *checksum, GCancellable *cancellable, GError **error) @@ -676,20 +676,24 @@ _import_payload_link (OstreeRepo *self, glnx_unref_object OtChecksumInstream *checksum_payload = NULL; g_autoptr(GFileInfo) file_info = NULL; - if (!_check_support_reflink (self, &reflinks_supported, error)) + /* The two repositories are on different devices */ + if (src_repo->device != dest_repo->device) + return TRUE; + + if (!_check_support_reflink (dest_repo, &reflinks_supported, error)) return FALSE; if (!reflinks_supported) return TRUE; - if (!G_IN_SET(self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER, OSTREE_REPO_MODE_BARE_USER_ONLY)) + if (!G_IN_SET(dest_repo->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER, OSTREE_REPO_MODE_BARE_USER_ONLY)) return TRUE; - if (!ostree_repo_load_file (source, checksum, &is, &file_info, NULL, cancellable, error)) + if (!ostree_repo_load_file (src_repo, checksum, &is, &file_info, NULL, cancellable, error)) return FALSE; if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR - || g_file_info_get_size (file_info) < self->payload_link_threshold) + || g_file_info_get_size (file_info) < dest_repo->payload_link_threshold) return TRUE; checksum_payload = ot_checksum_instream_new (is, G_CHECKSUM_SHA256); @@ -706,11 +710,12 @@ _import_payload_link (OstreeRepo *self, } payload_checksum = ot_checksum_instream_get_string (checksum_payload); - return _create_payload_link (self, checksum, payload_checksum, file_info, cancellable, error); + return _create_payload_link (dest_repo, checksum, payload_checksum, file_info, cancellable, error); } static gboolean _try_clone_from_payload_link (OstreeRepo *self, + OstreeRepo *dest_repo, const char *payload_checksum, GFileInfo *file_info, GLnxTmpfile *tmpf, @@ -722,7 +727,11 @@ _try_clone_from_payload_link (OstreeRepo *self, if (self->commit_stagedir.initialized) dfd_searches[0] = self->commit_stagedir.fd; - if (!_check_support_reflink (self, &reflinks_supported, error)) + /* The two repositories are on different devices */ + if (self->device != dest_repo->device) + return TRUE; + + if (!_check_support_reflink (dest_repo, &reflinks_supported, error)) return FALSE; if (!reflinks_supported) @@ -777,6 +786,8 @@ _try_clone_from_payload_link (OstreeRepo *self, return TRUE; } } + if (self->parent_repo) + return _try_clone_from_payload_link (self->parent_repo, dest_repo, payload_checksum, file_info, tmpf, cancellable, error); return TRUE; } @@ -1071,7 +1082,7 @@ write_content_object (OstreeRepo *self, /* Check if a file with the same payload is present in the repository, and in case try to reflink it */ - if (actual_payload_checksum && !_try_clone_from_payload_link (self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) + if (actual_payload_checksum && !_try_clone_from_payload_link (self, self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) return FALSE; /* This path is for regular files */ diff --git a/tests/installed/nondestructive/itest-payload-link.sh b/tests/installed/nondestructive/itest-payload-link.sh index 62b84667..6a6a01d3 100755 --- a/tests/installed/nondestructive/itest-payload-link.sh +++ b/tests/installed/nondestructive/itest-payload-link.sh @@ -30,6 +30,9 @@ date # Use /var/tmp so we have O_TMPFILE etc. prepare_tmpdir /var/tmp trap _tmpdir_cleanup EXIT +# We use this user down below, it needs access too +setfacl -d -m u:bin:rwX . +setfacl -m u:bin:rwX . ostree --repo=repo init --mode=archive echo -e '[archive]\nzlib-level=1\n' >> repo/config @@ -49,23 +52,40 @@ origin=$(cat ${test_tmpdir}/httpd-address) cleanup() { cd ${test_tmpdir} - umount mnt || true - test -n "${blkdev:-}" && losetup -d ${blkdev} || true + for mnt in ${mnts:-}; do + umount ${mnt} || true + done + for blkdev in ${blkdevs:-}; do + losetup -d ${blkdev} || true + done } trap cleanup EXIT -mkdir mnt -truncate -s 2G testblk.img -if ! blkdev=$(losetup --find --show $(pwd)/testblk.img); then - echo "ok # SKIP not run when cannot setup loop device" - exit 0 -fi - +truncate -s 2G testblk1.img +blkdev1=$(losetup --find --show $(pwd)/testblk1.img) +blkdevs="${blkdev1}" # This filesystem must support reflinks -mkfs.xfs -m reflink=1 ${blkdev} +mkfs.xfs -m reflink=1 ${blkdev1} +mkdir mnt1 +mount ${blkdev1} mnt1 +mnts=mnt1 -mount ${blkdev} mnt -cd mnt +truncate -s 2G testblk2.img +blkdev2=$(losetup --find --show $(pwd)/testblk2.img) +blkdevs="${blkdev1} ${blkdev2}" +mkfs.xfs -m reflink=1 ${blkdev2} +mkdir mnt2 +mount ${blkdev2} mnt2 +mnts="mnt1 mnt2" + +cd mnt1 +# See above for setfacl rationale +setfacl -d -m u:bin:rwX . +setfacl -m u:bin:rwX . +ls -al . +runuser -u bin mkdir foo +runuser -u bin touch foo/bar +ls -al foo # Test that reflink is really there (not just --reflink=auto) touch a @@ -78,13 +98,44 @@ ostree --repo=repo pull --disable-static-deltas origin dupobjects find repo -type l -name '*.payload-link' >payload-links.txt assert_streq "$(wc -l < payload-links.txt)" "1" -# Disable logging for inner loop -set +x cat payload-links.txt | while read i; do payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1) assert_streq "${payload_checksum}" "${payload_checksum_calculated}" done -set -x -echo "ok pull creates .payload-link" +echo "ok payload link" + +ostree --repo=repo checkout dupobjects content +# And another object which differs just in metadata +cp --reflink=auto content/bigobject{,3} +chown operator:0 content/bigobject3 +cat >unpriv-child-repo.sh <payload-links.txt +assert_streq "$(wc -l < payload-links.txt)" "0" +rm content -rf + +echo "ok reflink unprivileged with parent repo" + +# We can't reflink across devices though +cd ../mnt2 +ostree --repo=repo init --mode=archive +ostree --repo=repo config set core.parent $(cd ../mnt1/repo && pwd) +ostree --repo=../mnt1/repo checkout dupobjects content +ostree --repo=repo commit -b dupobjects2 --consume --tree=dir=content +ostree --repo=repo pull --disable-static-deltas origin dupobjects +find repo -type l -name '*.payload-link' >payload-links.txt +assert_streq "$(wc -l < payload-links.txt)" "0" + +echo "ok payload link across devices" + date From f258e9e5ff25e5c814bb54e4eb27e34841e913e5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 6 Apr 2018 15:37:45 +0200 Subject: [PATCH 32/80] lib/repo: Add ostree_repo_traverse_commit_union_with_parents This is a version of ostree_repo_traverse_commit_union that also remembers where the objects came from, by recording the parent relationships in a hashtable. This can be used to later find which commits each object was from, which we want to use in fsck. Closes: #1533 Approved by: cgwalters --- apidoc/ostree-sections.txt | 3 + src/libostree/libostree-devel.sym | 3 + src/libostree/ostree-repo-traverse.c | 189 +++++++++++++++++++++++++-- src/libostree/ostree-repo.h | 14 ++ 4 files changed, 195 insertions(+), 14 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 94206e03..5162b2f7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -396,8 +396,11 @@ OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate ostree_repo_static_delta_execute_offline ostree_repo_traverse_new_reachable +ostree_repo_traverse_new_parents +ostree_repo_traverse_parents_get_commits ostree_repo_traverse_commit ostree_repo_traverse_commit_union +ostree_repo_traverse_commit_union_with_parents ostree_repo_commit_traverse_iter_cleanup ostree_repo_commit_traverse_iter_clear ostree_repo_commit_traverse_iter_get_dir diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 07e11cb6..226b2ead 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -22,6 +22,9 @@ LIBOSTREE_2018.5 { ostree_sysroot_stage_tree; ostree_sysroot_get_staged_deployment; ostree_deployment_is_staged; + ostree_repo_traverse_new_parents; + ostree_repo_traverse_parents_get_commits; + ostree_repo_traverse_commit_union_with_parents; } LIBOSTREE_2018.3; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-traverse.c b/src/libostree/ostree-repo-traverse.c index be3ae109..d0edd65d 100644 --- a/src/libostree/ostree-repo-traverse.c +++ b/src/libostree/ostree-repo-traverse.c @@ -294,18 +294,141 @@ ostree_repo_traverse_new_reachable (void) NULL, (GDestroyNotify)g_variant_unref); } +/** + * ostree_repo_traverse_new_parents: + * + * This hash table is a mapping from #GVariant which can be accessed + * via ostree_object_name_deserialize() to a #GVariant containing either + * a similar #GVariant or and array of them, listing the parents of the key. + * + * Returns: (transfer container) (element-type GVariant GVariant): A new hash table + * + * Since: 2018.5 + */ +GHashTable * +ostree_repo_traverse_new_parents (void) +{ + return g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify)g_variant_unref, (GDestroyNotify)g_variant_unref); +} + +static void +parents_get_commits (GHashTable *parents_ht, GVariant *object, GHashTable *res) +{ + const char *checksum; + OstreeObjectType type; + + if (object == NULL) + return; + + ostree_object_name_deserialize (object, &checksum, &type); + if (type == OSTREE_OBJECT_TYPE_COMMIT) + g_hash_table_add (res, g_strdup (checksum)); + else + { + GVariant *parents = g_hash_table_lookup (parents_ht, object); + + if (parents == NULL) + g_debug ("Unexpected NULL parent"); + else if (g_variant_is_of_type (parents, G_VARIANT_TYPE_ARRAY)) + { + gsize i, len = g_variant_n_children (parents); + + for (i = 0; i < len; i++) + { + g_autoptr(GVariant) parent = g_variant_get_child_value (parents, i); + parents_get_commits (parents_ht, parent, res); + } + } + else + parents_get_commits (parents_ht, parents, res); + } +} + +/** + * ostree_repo_traverse_parents_get_commits: + * + * Gets all the commits that a certain object belongs to, as recorded + * by a parents table gotten from ostree_repo_traverse_commit_union_with_parents. + * + * Returns: (transfer full) (array zero-terminated=1): An array of checksums for + * the commits the key belongs to. + * + * Since: 2018.5 + */ +char ** +ostree_repo_traverse_parents_get_commits (GHashTable *parents, GVariant *object) +{ + g_autoptr(GHashTable) res = g_hash_table_new (g_str_hash, g_str_equal); + + parents_get_commits (parents, object, res); + + return (char **)g_hash_table_get_keys_as_array (res, NULL); +} + static gboolean traverse_dirtree (OstreeRepo *repo, const char *checksum, + GVariant *parent_key, GHashTable *inout_reachable, + GHashTable *inout_parents, gboolean ignore_missing_dirs, GCancellable *cancellable, GError **error); +static void +add_parent_ref (GHashTable *inout_parents, + GVariant *key, + GVariant *parent_key) +{ + GVariant *old_parents; + + if (inout_parents == NULL) + return; + + old_parents = g_hash_table_lookup (inout_parents, key); + if (old_parents == NULL) + { + /* For the common case of a single pointer we skip using an array to save memory. */ + g_hash_table_insert (inout_parents, g_variant_ref (key), g_variant_ref (parent_key)); + } + else + { + g_autofree GVariant **new_parents = NULL; + gsize i, len = 0; + + if (g_variant_is_of_type (old_parents, G_VARIANT_TYPE_ARRAY)) + { + gsize old_parents_len = g_variant_n_children (old_parents); + new_parents = g_new (GVariant *, old_parents_len + 1); + for (i = 0; i < old_parents_len ; i++) + { + g_autoptr(GVariant) old_parent = g_variant_get_child_value (old_parents, i); + if (!g_variant_equal (old_parent, parent_key)) + new_parents[len++] = g_steal_pointer (&old_parent); + } + } + else + { + new_parents = g_new (GVariant *, 2); + if (!g_variant_equal (old_parents, parent_key)) + new_parents[len++] = g_variant_ref (old_parents); + } + new_parents[len++] = g_variant_ref (parent_key); + g_hash_table_insert (inout_parents, g_variant_ref (key), + g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(su)"), new_parents , len))); + for (i = 0; i < len; i++) + g_variant_unref (new_parents[i]); + } +} + + static gboolean traverse_iter (OstreeRepo *repo, OstreeRepoCommitTraverseIter *iter, + GVariant *parent_key, GHashTable *inout_reachable, + GHashTable *inout_parents, gboolean ignore_missing_dirs, GCancellable *cancellable, GError **error) @@ -343,6 +466,7 @@ traverse_iter (OstreeRepo *repo, g_debug ("Found file object %s", checksum); key = g_variant_ref_sink (ostree_object_name_serialize (checksum, OSTREE_OBJECT_TYPE_FILE)); + add_parent_ref (inout_parents, key, parent_key); g_hash_table_add (inout_reachable, g_steal_pointer (&key)); } else if (iterres == OSTREE_REPO_COMMIT_ITER_RESULT_DIR) @@ -357,16 +481,18 @@ traverse_iter (OstreeRepo *repo, g_debug ("Found dirtree object %s", content_checksum); g_debug ("Found dirmeta object %s", meta_checksum); key = g_variant_ref_sink (ostree_object_name_serialize (meta_checksum, OSTREE_OBJECT_TYPE_DIR_META)); + add_parent_ref (inout_parents, key, parent_key); g_hash_table_add (inout_reachable, g_steal_pointer (&key)); key = g_variant_ref_sink (ostree_object_name_serialize (content_checksum, OSTREE_OBJECT_TYPE_DIR_TREE)); + add_parent_ref (inout_parents, key, parent_key); if (!g_hash_table_lookup (inout_reachable, key)) { - g_hash_table_add (inout_reachable, g_steal_pointer (&key)); - - if (!traverse_dirtree (repo, content_checksum, inout_reachable, + if (!traverse_dirtree (repo, content_checksum, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error)) return FALSE; + + g_hash_table_add (inout_reachable, g_steal_pointer (&key)); } } else @@ -379,7 +505,9 @@ traverse_iter (OstreeRepo *repo, static gboolean traverse_dirtree (OstreeRepo *repo, const char *checksum, + GVariant *parent_key, GHashTable *inout_reachable, + GHashTable *inout_parents, gboolean ignore_missing_dirs, GCancellable *cancellable, GError **error) @@ -409,31 +537,39 @@ traverse_dirtree (OstreeRepo *repo, error)) return FALSE; - if (!traverse_iter (repo, &iter, inout_reachable, ignore_missing_dirs, cancellable, error)) + if (!traverse_iter (repo, &iter, parent_key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error)) return FALSE; return TRUE; } /** - * ostree_repo_traverse_commit_union: (skip) + * ostree_repo_traverse_commit_union_with_parents: (skip) * @repo: Repo * @commit_checksum: ASCII SHA256 checksum * @maxdepth: Traverse this many parent commits, -1 for unlimited * @inout_reachable: Set of reachable objects + * @inout_parents: Map from object to parent object * @cancellable: Cancellable * @error: Error * * Update the set @inout_reachable containing all objects reachable * from @commit_checksum, traversing @maxdepth parent commits. + * + * Additionally this constructs a mapping from each object to the parents + * of the object, which can be used to track which commits an object + * belongs to. + * + * Since: 2018.5 */ gboolean -ostree_repo_traverse_commit_union (OstreeRepo *repo, - const char *commit_checksum, - int maxdepth, - GHashTable *inout_reachable, - GCancellable *cancellable, - GError **error) +ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GHashTable *inout_parents, + GCancellable *cancellable, + GError **error) { g_autofree char *tmp_checksum = NULL; @@ -467,8 +603,7 @@ ostree_repo_traverse_commit_union (OstreeRepo *repo, if ((commitstate & OSTREE_REPO_COMMIT_STATE_PARTIAL) != 0) ignore_missing_dirs = TRUE; - g_hash_table_add (inout_reachable, key); - key = NULL; + g_hash_table_add (inout_reachable, g_variant_ref (key)); g_debug ("Traversing commit %s", commit_checksum); ostree_cleanup_repo_commit_traverse_iter @@ -478,7 +613,7 @@ ostree_repo_traverse_commit_union (OstreeRepo *repo, error)) return FALSE; - if (!traverse_iter (repo, &iter, inout_reachable, ignore_missing_dirs, cancellable, error)) + if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error)) return FALSE; gboolean recurse = FALSE; @@ -501,6 +636,32 @@ ostree_repo_traverse_commit_union (OstreeRepo *repo, return TRUE; } +/** + * ostree_repo_traverse_commit_union: (skip) + * @repo: Repo + * @commit_checksum: ASCII SHA256 checksum + * @maxdepth: Traverse this many parent commits, -1 for unlimited + * @inout_reachable: Set of reachable objects + * @cancellable: Cancellable + * @error: Error + * + * Update the set @inout_reachable containing all objects reachable + * from @commit_checksum, traversing @maxdepth parent commits. + */ +gboolean +ostree_repo_traverse_commit_union (OstreeRepo *repo, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GCancellable *cancellable, + GError **error) +{ + return + ostree_repo_traverse_commit_union_with_parents (repo, commit_checksum, maxdepth, + inout_reachable, NULL, + cancellable, error); +} + /** * ostree_repo_traverse_commit: * @repo: Repo diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index faac5d9a..04b04416 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1112,6 +1112,12 @@ gboolean ostree_repo_static_delta_execute_offline (OstreeRepo _OSTREE_PUBLIC GHashTable *ostree_repo_traverse_new_reachable (void); +_OSTREE_PUBLIC +GHashTable *ostree_repo_traverse_new_parents (void); + +_OSTREE_PUBLIC +char ** ostree_repo_traverse_parents_get_commits (GHashTable *parents, GVariant *object); + _OSTREE_PUBLIC gboolean ostree_repo_traverse_commit (OstreeRepo *repo, const char *commit_checksum, @@ -1127,6 +1133,14 @@ gboolean ostree_repo_traverse_commit_union (OstreeRepo *repo, GHashTable *inout_reachable, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GHashTable *inout_parents, + GCancellable *cancellable, + GError **error); struct _OstreeRepoCommitTraverseIter { gboolean initialized; From 474556b95547d75fe1cc4c2131a878489c7a3f82 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 6 Apr 2018 15:39:43 +0200 Subject: [PATCH 33/80] fsck: Mark commits with missing or deleted object partial This means we can later use various operations to heal the repository because ostree does not assume all objects are there. This the begining of a fix for https://github.com/ostreedev/ostree/pull/345 Closes: #1533 Approved by: cgwalters --- src/ostree/ot-builtin-fsck.c | 40 +++++++++++++++---- tests/installed/nondestructive/itest-pull.sh | 15 +++++++ tests/test-corruption.sh | 42 +++++++++++++++++++- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 3f9da783..8a44b619 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -53,6 +53,8 @@ static gboolean fsck_one_object (OstreeRepo *repo, const char *checksum, OstreeObjectType objtype, + GHashTable *object_parents, + GVariant *key, gboolean *out_found_corruption, GCancellable *cancellable, GError **error) @@ -60,12 +62,14 @@ fsck_one_object (OstreeRepo *repo, g_autoptr(GError) temp_error = NULL; if (!ostree_repo_fsck_object (repo, objtype, checksum, cancellable, &temp_error)) { + gboolean object_missing = FALSE; + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { g_clear_error (&temp_error); g_printerr ("Object missing: %s.%s\n", checksum, ostree_object_type_to_string (objtype)); - *out_found_corruption = TRUE; + object_missing = TRUE; } else { @@ -73,7 +77,7 @@ fsck_one_object (OstreeRepo *repo, { g_printerr ("%s\n", temp_error->message); (void) ostree_repo_delete_object (repo, objtype, checksum, cancellable, NULL); - *out_found_corruption = TRUE; + object_missing = TRUE; } else { @@ -81,6 +85,26 @@ fsck_one_object (OstreeRepo *repo, return FALSE; } } + + if (object_missing) + { + *out_found_corruption = TRUE; + + if (object_parents != NULL && objtype != OSTREE_OBJECT_TYPE_COMMIT) + { + g_auto(GStrv) parent_commits = ostree_repo_traverse_parents_get_commits (object_parents, key); + int i; + + /* The commit was missing or deleted, mark the commit partial */ + for (i = 0; parent_commits[i] != NULL; i++) + { + const char *parent_commit = parent_commits[i]; + g_printerr ("Marking commit %s as partial\n", parent_commit); + if (!ostree_repo_mark_commit_partial (repo, parent_commit, TRUE, error)) + return FALSE; + } + } + } } return TRUE; @@ -94,6 +118,7 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, GError **error) { g_autoptr(GHashTable) reachable_objects = ostree_repo_traverse_new_reachable (); + g_autoptr(GHashTable) object_parents = ostree_repo_traverse_new_parents (); GHashTableIter hash_iter; gpointer key, value; @@ -108,8 +133,8 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, g_assert (objtype == OSTREE_OBJECT_TYPE_COMMIT); - if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable_objects, - cancellable, error)) + if (!ostree_repo_traverse_commit_union_with_parents (repo, checksum, 0, reachable_objects, object_parents, + cancellable, error)) return FALSE; } @@ -127,8 +152,9 @@ fsck_reachable_objects_from_commits (OstreeRepo *repo, ostree_object_name_deserialize (serialized_key, &checksum, &objtype); - if (!fsck_one_object (repo, checksum, objtype, out_found_corruption, - cancellable, error)) + if (!fsck_one_object (repo, checksum, objtype, + object_parents, serialized_key, + out_found_corruption, cancellable, error)) return FALSE; i++; @@ -150,7 +176,7 @@ fsck_commit_for_ref (OstreeRepo *repo, GError **error) { if (!fsck_one_object (repo, checksum, OSTREE_OBJECT_TYPE_COMMIT, - found_corruption, + NULL, NULL, found_corruption, cancellable, error)) return FALSE; diff --git a/tests/installed/nondestructive/itest-pull.sh b/tests/installed/nondestructive/itest-pull.sh index a7cd922d..b3a52a27 100755 --- a/tests/installed/nondestructive/itest-pull.sh +++ b/tests/installed/nondestructive/itest-pull.sh @@ -30,6 +30,20 @@ ostree --repo=bare-repo init --mode=bare-user ostree --repo=bare-repo remote add origin --set=gpg-verify=false $(cat ${test_tmpdir}/httpd-address) log_timestamps ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref} +echo "ok pull" + +# fsck marks commits partial +# https://github.com/ostreedev/ostree/pull/1533 +for d in $(find bare-repo/objects/ -maxdepth 1 -type d); do + (find ${d} -name '*.file' || true) | head -20 | xargs rm -vf +done +if ostree --repo=bare-repo fsck; then + fatal "fsck unexpectedly succeeded" +fi +ostree --repo=bare-repo pull origin ${host_nonremoteref} +# Don't need a full fsck here +ostree --repo=bare-repo ls origin:${host_nonremoteref} >/dev/null + rm bare-repo repo -rf # Try copying the host's repo across a mountpoint for direct @@ -44,6 +58,7 @@ log_timestamps ostree --repo=repo fsck cd .. umount mnt +# Cleanup kill -TERM $(cat ${test_tmpdir}/httpd-pid) echo "ok" date diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index 997d39c7..3b4a649e 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..6" +echo "1..8" . $(dirname $0)/libtest.sh @@ -89,3 +89,43 @@ if ${CMD_PREFIX} ostree --repo=ostree-path-traverse/repo checkout pathtraverse-t fi assert_file_has_content_literal err.txt 'Invalid / in filename ../afile' echo "ok path traverse checkout" + +cd ${test_tmpdir} +rm repo files -rf +setup_test_repository "bare" + +rev=$($OSTREE rev-parse test2) + +filechecksum=$(ostree_file_path_to_checksum repo test2 /firstfile) +rm repo/$(ostree_checksum_to_relative_object_path repo $filechecksum) + +assert_not_has_file repo/state/${rev}.commitpartial + +if $OSTREE fsck -q 2>err.txt; then + assert_not_reached "fsck unexpectedly succeeded" +fi +assert_file_has_content_literal err.txt "Object missing:" +assert_file_has_content_literal err.txt "Marking commit $rev as partial" +assert_has_file repo/state/${rev}.commitpartial + +echo "ok missing file" + +cd ${test_tmpdir} +rm repo files -rf +setup_test_repository "bare" + +rev=$($OSTREE rev-parse test2) + +filechecksum=$(ostree_file_path_to_checksum repo test2 /firstfile) +echo corrupted >> repo/$(ostree_checksum_to_relative_object_path repo $filechecksum) + +assert_not_has_file repo/state/${rev}.commitpartial + +if $OSTREE fsck -q --delete 2>err.txt; then + assert_not_reached "fsck unexpectedly succeeded" +fi +assert_file_has_content_literal err.txt "Corrupted file object;" +assert_file_has_content_literal err.txt "Marking commit $rev as partial" +assert_has_file repo/state/${rev}.commitpartial + +echo "ok corrupt file" From 09dc2a87724111e83249b42dd4c4faa5dd2c2840 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 17 Apr 2018 15:21:04 -0400 Subject: [PATCH 34/80] ci: use gpgme scratch build to workaround issues Temporarily work around gpgme issues until the package gets into stable. For more information, see: https://src.fedoraproject.org/rpms/gpgme/pull-request/3 https://github.com/ostreedev/ostree/pull/1539 Closes: #1540 Approved by: cgwalters --- ci/libpaprci/libbuild.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh index 074494f0..9ad417b2 100644 --- a/ci/libpaprci/libbuild.sh +++ b/ci/libpaprci/libbuild.sh @@ -60,6 +60,12 @@ pkg_builddep() { # This is sadly the only case where it's a different command if test -x /usr/bin/dnf; then dnf builddep -y "$@" + + # XXX: tmp hack -- see + # https://github.com/ostreedev/ostree/pull/1539 + if rpm -q gpgme | grep -q gpgme-1.9.0-6.fc27; then + dnf install -y https://kojipkgs.fedoraproject.org//packages/gpgme/1.10.0/4.fc27/x86_64/{gpgme{,-devel},python{2,3}-gpg}-1.10.0-4.fc27.x86_64.rpm + fi else yum-builddep -y "$@" fi From 16d3359bf8827f6a6d70aa313c6e7f16a4f2e277 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 12 Apr 2018 12:40:08 -0400 Subject: [PATCH 35/80] lib/sysroot: Move staged into deployment list, rework handling Followup to: https://github.com/ostreedev/ostree/pull/1503 After starting some more work on on this in rpm-ostree, it is actually simpler if the staged deployment just shows up in the list. It's effectively opt-in today; down the line we may make it the default, but I worry about breaking things that e.g. assume they can mutate the deployment before rebooting and have `/etc` already merged. There's not that many things in libostree that iterate over the deployment list. The biggest change here is around the `ostree_sysroot_write_deployments_with_options` API. I initially tried hard to support a use case like "push a rollback" while retaining the staged deployment, but everything gets very messy because that function truly is operating on the bootloader list. For now what I settled on is to just discard the staged deployment; down the line we can enhance things. Where we then have some new gymnastics is around implementing the finalization; we need to go to some effort to pull the staged deployment out of the list and mark it as unstaged, and then pass it down to `write_deployments()`. Closes: #1539 Approved by: jlebon --- src/libostree/ostree-sysroot-cleanup.c | 9 --- src/libostree/ostree-sysroot-deploy.c | 57 +++++++++++++++++-- src/libostree/ostree-sysroot.c | 31 ++++++++-- src/ostree/ot-admin-builtin-status.c | 10 ---- tests/installed/destructive/staged-deploy.yml | 38 ++++++++++++- 5 files changed, 113 insertions(+), 32 deletions(-) diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 3698767f..1d46222b 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -308,15 +308,6 @@ cleanup_old_deployments (OstreeSysroot *self, g_hash_table_replace (active_boot_checksums, bootcsum, bootcsum); } - /* And also the staged deployment, if any */ - if (self->staged_deployment) - { - char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, self->staged_deployment); - g_hash_table_replace (active_deployment_dirs, deployment_path, deployment_path); - char *bootcsum = g_strdup (ostree_deployment_get_bootcsum (self->staged_deployment)); - g_hash_table_replace (active_boot_checksums, bootcsum, bootcsum); - } - /* Find all deployment directories, both active and inactive */ g_autoptr(GPtrArray) all_deployment_dirs = NULL; if (!list_all_deployment_directories (self, &all_deployment_dirs, diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index e7dc25d0..3843a89e 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2144,6 +2144,38 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, { g_assert (self->loaded); + /* It dramatically simplifies a lot of the logic below if we + * drop the staged deployment from both the source deployment list, + * as well as the target list. We don't want to write it to the bootloader + * now, which is mostly what this function is concerned with. + * In the future we though should probably adapt things to keep it. + */ + if (self->staged_deployment) + { + if (!glnx_unlinkat (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED, 0, error)) + return FALSE; + + if (!_ostree_sysroot_rmrf_deployment (self, self->staged_deployment, cancellable, error)) + return FALSE; + + g_assert (self->staged_deployment == self->deployments->pdata[0]); + g_ptr_array_remove_index (self->deployments, 0); + } + /* First new deployment; we'll see if it's staged */ + OstreeDeployment *first_new = + (new_deployments->len > 0 ? new_deployments->pdata[0] : NULL); + g_autoptr(GPtrArray) new_deployments_copy = NULL; + if (first_new && ostree_deployment_is_staged (first_new)) + { + g_assert_cmpint (new_deployments->len, >, 0); + new_deployments_copy = g_ptr_array_sized_new (new_deployments->len - 1); + for (guint i = 1; i < new_deployments->len; i++) + g_ptr_array_add (new_deployments_copy, new_deployments->pdata[i]); + } + else + new_deployments_copy = g_ptr_array_ref (new_deployments); + new_deployments = new_deployments_copy; + /* Assign a bootserial to each new deployment. */ assign_bootserials (new_deployments); @@ -2162,6 +2194,8 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, for (guint i = 0; i < new_deployments->len; i++) { OstreeDeployment *cur_deploy = self->deployments->pdata[i]; + if (ostree_deployment_is_staged (cur_deploy)) + continue; OstreeDeployment *new_deploy = new_deployments->pdata[i]; if (!deployment_bootconfigs_equal (cur_deploy, new_deploy)) { @@ -2185,12 +2219,12 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, for (guint i = 0; i < new_deployments->len; i++) { OstreeDeployment *deployment = new_deployments->pdata[i]; - g_autoptr(GFile) deployment_root = NULL; + g_assert (!ostree_deployment_is_staged (deployment)); if (deployment == self->booted_deployment) found_booted_deployment = TRUE; - deployment_root = ostree_sysroot_get_deployment_directory (self, deployment); + g_autoptr(GFile) deployment_root = ostree_sysroot_get_deployment_directory (self, deployment); if (!g_file_query_exists (deployment_root, NULL)) return glnx_throw (error, "Unable to find expected deployment root: %s", gs_file_get_path_cached (deployment_root)); @@ -2672,7 +2706,10 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, return FALSE; } - if (!_ostree_sysroot_reload_staged (self, error)) + /* Bump mtime so external processes know something changed, and then reload. */ + if (!_ostree_sysroot_bump_mtime (self, error)) + return FALSE; + if (!ostree_sysroot_load (self, cancellable, error)) return FALSE; return TRUE; @@ -2716,10 +2753,17 @@ _ostree_sysroot_finalize_staged (OstreeSysroot *self, cancellable, error)) return FALSE; + /* Now, take ownership of the staged state, as normally the API below strips + * it out. + */ + g_autoptr(OstreeDeployment) staged = g_steal_pointer (&self->staged_deployment); + staged->staged = FALSE; + g_ptr_array_remove_index (self->deployments, 0); + /* TODO: Proxy across flags too? */ OstreeSysrootSimpleWriteDeploymentFlags flags = 0; - if (!ostree_sysroot_simple_write_deployment (self, ostree_deployment_get_osname (self->staged_deployment), - self->staged_deployment, merge_deployment, flags, + if (!ostree_sysroot_simple_write_deployment (self, ostree_deployment_get_osname (staged), + staged, merge_deployment, flags, cancellable, error)) return FALSE; @@ -2744,6 +2788,9 @@ ostree_sysroot_deployment_set_kargs (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + /* For now; instead of this do a redeployment */ + g_assert (!ostree_deployment_is_staged (deployment)); + g_autoptr(OstreeDeployment) new_deployment = ostree_deployment_clone (deployment); OstreeBootconfigParser *new_bootconfig = ostree_deployment_get_bootconfig (new_deployment); diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 51d51340..f59f1508 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -740,6 +740,15 @@ compare_deployments_by_boot_loader_version_reversed (gconstpointer a_pp, OstreeBootconfigParser *a_bootconfig = ostree_deployment_get_bootconfig (a); OstreeBootconfigParser *b_bootconfig = ostree_deployment_get_bootconfig (b); + /* Staged deployments are always first */ + if (ostree_deployment_is_staged (a)) + { + g_assert (!ostree_deployment_is_staged (b)); + return -1; + } + else if (ostree_deployment_is_staged (b)) + return 1; + return compare_boot_loader_configs (a_bootconfig, b_bootconfig); } @@ -824,11 +833,16 @@ _ostree_sysroot_reload_staged (OstreeSysroot *self, g_variant_dict_lookup (staged_deployment_dict, "kargs", "^a&s", &kargs); if (target) { - self->staged_deployment = + g_autoptr(OstreeDeployment) staged = _ostree_sysroot_deserialize_deployment_from_variant (target, error); - if (!self->staged_deployment) + if (!staged) return FALSE; - _ostree_deployment_set_bootconfig_from_kargs (self->staged_deployment, kargs); + + _ostree_deployment_set_bootconfig_from_kargs (staged, kargs); + if (!load_origin (self, staged, NULL, error)) + return FALSE; + + self->staged_deployment = g_steal_pointer (&staged); self->staged_deployment_data = g_steal_pointer (&staged_deployment_data); /* We set this flag for ostree_deployment_is_staged() because that API * doesn't have access to the sysroot, which currently has the @@ -938,8 +952,16 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (root_is_ostree_booted && !self->booted_deployment) return glnx_throw (error, "Unexpected state: /run/ostree-booted found and in / sysroot but not in a booted deployment"); + if (!_ostree_sysroot_reload_staged (self, error)) + return FALSE; + /* Ensure the entires are sorted */ g_ptr_array_sort (deployments, compare_deployments_by_boot_loader_version_reversed); + + /* Staged shows up first */ + if (self->staged_deployment) + g_ptr_array_insert (deployments, 0, g_object_ref (self->staged_deployment)); + /* And then set their index variables */ for (guint i = 0; i < deployments->len; i++) { @@ -947,9 +969,6 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, ostree_deployment_set_index (deployment, i); } - if (!_ostree_sysroot_reload_staged (self, error)) - return FALSE; - /* Determine whether we're "physical" or not, the first time we initialize */ if (!self->loaded) { diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 55be6994..ca165f45 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -201,16 +201,6 @@ ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocat } else { - OstreeDeployment *staged = ostree_sysroot_get_staged_deployment (sysroot); - if (staged) - { - if (!deployment_print_status (sysroot, repo, staged, - FALSE, FALSE, FALSE, - cancellable, - error)) - return FALSE; - } - for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index bf504675..3f4062b2 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -1,9 +1,11 @@ -# Test the deploy --stage functionality +# Test the deploy --stage functionality; first, we stage a deployment +# reboot, and validate that it worked. - name: Write staged-deploy commit shell: | ostree --repo=/ostree/repo commit --parent="${commit}" -b staged-deploy --tree=ref="${commit}" --no-bindings ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + test -f /run/ostree/staged-deployment environment: commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" - include_tasks: ../tasks/reboot.yml @@ -13,12 +15,44 @@ journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' # And that we have the new kernel argument grep -q -e 'ostreetest=yes' /proc/cmdline + # And there should not be a staged deployment + test '!' -f /run/ostree/staged-deployment - name: Rollback shell: rpm-ostree rollback - include_tasks: ../tasks/reboot.yml - shell: | - ostree refs --delete staged-deploy rpm-ostree cleanup -rp # And now we shouldn't have the kernel commandline entry - name: Check we do not have new kernel cmdline entry shell: grep -qv -e 'ostreetest=yes' /proc/cmdline + +# Ensure we can unstage +- name: Write staged-deploy commit, then unstage + shell: | + ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + ostree admin status > status.txt + grep -qFe '(staged)' status.txt + test -f /run/ostree/staged-deployment + ostree admin undeploy 0 + ostree admin status > status.txt + grep -vqFe '(staged)' status.txt + test '!' -f /run/ostree/staged-deployment + environment: + commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" + +# Staged should be overwritten by non-staged +- name: Write staged-deploy commit, then unstage + shell: | + ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + test -f /run/ostree/staged-deployment + ostree --repo=/ostree/repo commit --parent="${commit}" -b nonstaged-deploy --tree=ref="${commit}" --no-bindings + ostree admin deploy --karg-proc-cmdline --karg=ostreetest=yes nonstaged-deploy + ostree admin status > status.txt + grep -vqFe '(staged)' status.txt + test '!' -f /run/ostree/staged-deployment + ostree admin undeploy 0 + environment: + commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" + +- name: Cleanup refs + shell: ostree refs --delete staged-deploy nonstaged-deploy From 002e5eeac605093e1f3935f10f8ae30fd4a2e659 Mon Sep 17 00:00:00 2001 From: Sam Spilsbury Date: Tue, 16 Jan 2018 09:23:11 +0800 Subject: [PATCH 36/80] avahi: Don't complain with g_warning if the daemon wasn't running This is a normal case when running unit tests in client code on continuous integration infrastructure. When those tests are running they will set G_DEBUG=fatal-warnings which will cause the program to abort if a warning is emitted. Instead, emit a debug message if the problem was that we couldn't connect to the daemon. Closes: #1542 Approved by: jlebon --- src/libostree/ostree-repo-finder-avahi.c | 12 +++++++++--- src/libostree/ostree-repo-pull.c | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index 514351fc..1e77a6e0 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -1437,9 +1437,15 @@ ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, if (client == NULL) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to create finder client: %s", - avahi_strerror (failure)); + if (failure == AVAHI_ERR_NO_DAEMON) + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Avahi daemon is not running: %s", + avahi_strerror (failure)); + else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to create finder client: %s", + avahi_strerror (failure)); + return; } diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index be7cb228..f5745f86 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4686,7 +4686,20 @@ ostree_repo_find_remotes_async (OstreeRepo *self, if (local_error != NULL) { - g_warning ("Avahi finder failed; removing it: %s", local_error->message); + /* See ostree-repo-finder-avahi.c:ostree_repo_finder_avahi_start, we + * intentionally throw this so as to distinguish between the Avahi + * finder failing because the Avahi daemon wasn't running and + * the Avahi finder failing because of some actual error. + * + * We need to distinguish between g_debug and g_warning here because + * unit tests that use this code may set G_DEBUG=fatal-warnings which + * would cause client code to abort if a warning were emitted. + */ + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + g_debug ("Avahi finder failed under normal operation; removing it: %s", local_error->message); + else + g_warning ("Avahi finder failed abnormally; removing it: %s", local_error->message); + default_finders[2] = NULL; g_clear_object (&finder_avahi); } From 653be1556d56e4d796254e2f89155bcbca840fa9 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Sun, 25 Mar 2018 23:57:38 -0700 Subject: [PATCH 37/80] tests: Fix typo in unit test Closes: #1543 Approved by: cgwalters --- tests/test-create-usb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-create-usb.sh b/tests/test-create-usb.sh index e057e90f..310385d9 100755 --- a/tests/test-create-usb.sh +++ b/tests/test-create-usb.sh @@ -94,7 +94,7 @@ assert_has_dir "dest-mount3/some-dest" assert_symlink_has_content "dest-mount3/.ostree/repos.d/00-generated" "/some-dest$" ${CMD_PREFIX} ostree --repo=dest-mount3/.ostree/repos.d/00-generated refs --collections > dest-refs assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$" -assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$" +assert_file_has_content dest-refs "^(org.example.Collection1, test-2)$" assert_file_has_content dest-refs "^(org.example.Collection1, test-3)$" assert_has_file dest-mount3/.ostree/repos.d/00-generated/summary From cf7888840130d1fe718c36f2a1155c06ffb90e92 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Mon, 26 Mar 2018 00:01:39 -0700 Subject: [PATCH 38/80] create-usb: Update summary in destination repo Currently the create-usb command only generates a summary file in the destination repo if one doesn't already exist, which means if one does exist it becomes out of date after the new refs are pulled. This commit makes ostree regenerate the summary regardless of whether it exists, so that consumers such as ostree_repo_find_remotes_async() (and at a higher level, GNOME Software) get an accurate picture of the refs available on the mount. This commit also updates one of the unit tests to check that the summary is accurate after a second pull into the same repo. Since any user of the create-usb command is using collection IDs they are new enough to be using the unsigned summary support. While it would technically be possible to use summary signatures on a repo and use the create-usb command on it (a scenario broken by this commit), the create-usb command is designed for P2P distribution of refs, which requires use of unsigned summary support. So this is a legitimate narrowing of the tool. Fixes https://github.com/ostreedev/ostree/issues/1465 Closes: #1543 Approved by: cgwalters --- src/ostree/ot-builtin-create-usb.c | 6 +----- tests/test-create-usb.sh | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-create-usb.c b/src/ostree/ot-builtin-create-usb.c index 02b55b94..07a214c6 100644 --- a/src/ostree/ot-builtin-create-usb.c +++ b/src/ostree/ot-builtin-create-usb.c @@ -201,11 +201,7 @@ ostree_builtin_create_usb (int argc, /* FIXME: It should be possible to work without this, but find_remotes_cb() in * ostree-repo-pull.c currently assumes a summary file (signed or unsigned) is * present. */ - struct stat stbuf; - if (!glnx_fstatat_allow_noent (ostree_repo_get_dfd (dest_repo), "summary", &stbuf, 0, error)) - return FALSE; - if (errno == ENOENT && - !ostree_repo_regenerate_summary (dest_repo, NULL, cancellable, error)) + if (!ostree_repo_regenerate_summary (dest_repo, NULL, cancellable, error)) return FALSE; /* Add the symlinks .ostree/repos.d/@symlink_name → @dest_repo_path, unless diff --git a/tests/test-create-usb.sh b/tests/test-create-usb.sh index 310385d9..392352fd 100755 --- a/tests/test-create-usb.sh +++ b/tests/test-create-usb.sh @@ -96,7 +96,10 @@ ${CMD_PREFIX} ostree --repo=dest-mount3/.ostree/repos.d/00-generated refs --coll assert_file_has_content dest-refs "^(org.example.Collection1, test-1)$" assert_file_has_content dest-refs "^(org.example.Collection1, test-2)$" assert_file_has_content dest-refs "^(org.example.Collection1, test-3)$" -assert_has_file dest-mount3/.ostree/repos.d/00-generated/summary +${CMD_PREFIX} ostree --repo=dest-mount3/.ostree/repos.d/00-generated summary -v > dest-summary +assert_file_has_content dest-summary "(org.example.Collection1, test-1)$" +assert_file_has_content dest-summary "(org.example.Collection1, test-2)$" +assert_file_has_content dest-summary "(org.example.Collection1, test-3)$" echo "ok 4 adding ref to an existing usb" From 972a1921f517f5b2ae0aafed42224161fb4bb6d8 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Tue, 17 Apr 2018 14:35:52 -0700 Subject: [PATCH 39/80] man: Add man page for create-usb Closes: #1543 Approved by: cgwalters --- Makefile-man.am | 2 +- man/ostree-create-usb.xml | 127 +++++++++++++++++++++++ man/ostree-find-remotes.xml | 6 ++ man/ostree.xml | 9 ++ src/libostree/ostree-repo-finder-mount.c | 3 +- src/ostree/ot-builtin-create-usb.c | 1 - 6 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 man/ostree-create-usb.xml diff --git a/Makefile-man.am b/Makefile-man.am index 4d99cde1..342af520 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -45,7 +45,7 @@ man1_files += rofiles-fuse.1 endif if ENABLE_EXPERIMENTAL_API -man1_files += ostree-find-remotes.1 +man1_files += ostree-find-remotes.1 ostree-create-usb.1 endif man5_files = ostree.repo.5 ostree.repo-config.5 diff --git a/man/ostree-create-usb.xml b/man/ostree-create-usb.xml new file mode 100644 index 00000000..d250d915 --- /dev/null +++ b/man/ostree-create-usb.xml @@ -0,0 +1,127 @@ + + + + + + + + + ostree create-usb + OSTree + + + + Developer + Matthew + Leeds + matthew.leeds@endlessm.com + + + + + + ostree create-usb + 1 + + + + ostree-create-usb + Put the given refs on an external drive for P2P distribution. + + + + + ostree create-usb OPTIONS MOUNT-PATH COLLECTION-ID REF COLLECTION-ID REF + + + + + Description + + + This command creates a repository in MOUNT-PATH and pulls the given + REF(s) into it so they can be found and pulled from (perhaps by another computer that's offline). + See + ostree-find-remotes1 + for more information on P2P distribution. + + + In order for ostree to pull refs from a mounted filesystem (such as + a USB drive) the repo must be in a standard location. Specifically, + subdirectories of .ostree/repos.d are checked, + then .ostree/repo, ostree/repo, + and var/lib/flatpak/repo are checked. By default + ostree create-usb uses .ostree/repo, + but if you specify another location using + a symbolic link will be created for you in .ostree/repos.d. + + + This command will regenerate the summary file + in the destination repo so that it stays accurate, so you shouldn't + try to use summary signatures in the destination repo. This + shouldn't be a concern because clients that support pulling from + USB mounts use signed per-repo and per-commit metadata instead of + summary signatures. + + + This command relies on the summary file in the source repo, so you + may want to run ostree summary -u before running + this command. + + + + + Options + + + + + + + Do not invoke fsync(). + + + + + + =DEST + + + Create the repository in DEST under MOUNT-PATH, rather than + the default location. + + + + + + + + Example + ostree --repo=/var/lib/flatpak/repo create-usb /run/media/mwleeds/f6d04c7a-60f6-4ba3-bb96-0f40498675be com.exampleos.Apps app/org.kde.Khangman/x86_64/stable com.exampleos.Apps ostree-metadata com.exampleos.Apps appstream/x86_64 + + +5 metadata, 213 content objects imported; 1 KiB transferred in 1 seconds Copied 3/3 refs successfully from ‘/var/lib/flatpak/repo’ to ‘.ostree/repo’ repository in ‘/run/media/mwleeds/f6d04c7a-60f6-4ba3-bb96-0f40498675be’. + + + + diff --git a/man/ostree-find-remotes.xml b/man/ostree-find-remotes.xml index f0208b7c..15f5a516 100644 --- a/man/ostree-find-remotes.xml +++ b/man/ostree-find-remotes.xml @@ -78,6 +78,12 @@ Boston, MA 02111-1307, USA. default set of finders (sources for remotes) using the option documented below. + + The create-usb command is the recommended way to + put refs on a USB such that find-remotes will + discover them. See + ostree-create-usb1. + diff --git a/man/ostree.xml b/man/ostree.xml index 06076c45..8a79cd04 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -283,6 +283,15 @@ Boston, MA 02111-1307, USA. + + ostree-diff1 diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 7339fe52..09e85035 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -434,7 +434,8 @@ G_GNUC_END_IGNORE_DEPRECATIONS g_array_sort (repos_refs, repo_and_refs_compare); /* Also check the well-known special-case directories in the mount. - * Add them after sorting, so they’re always last. */ + * Add them after sorting, so they’re always last. + * NOTE: If you change these, update the man page. */ const gchar * const well_known_repos[] = { ".ostree/repo", diff --git a/src/ostree/ot-builtin-create-usb.c b/src/ostree/ot-builtin-create-usb.c index 07a214c6..57feeaa1 100644 --- a/src/ostree/ot-builtin-create-usb.c +++ b/src/ostree/ot-builtin-create-usb.c @@ -41,7 +41,6 @@ static GOptionEntry options[] = { NULL } }; -/* TODO: Add a man page. */ gboolean ostree_builtin_create_usb (int argc, char **argv, From e5f6c9d1e2266a3c7bc523a432aa164123048bf0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 19 Apr 2018 15:53:58 -0400 Subject: [PATCH 40/80] tests/installed: Make reboot task less racy This took a whole lot of experimentation. I hit upon the idea of doing a `systemctl stop sshd` to avoid the situation where we might ssh back into the system while it's in the process of shutting down. Ultimately the other fix is disabling `ControlMaster`; see for example: https://github.com/ansible/ansible/issues/17935 Closes: #1548 Approved by: cgwalters --- tests/installed/playbook-run.sh | 2 ++ tests/installed/tasks/reboot.yml | 37 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/installed/playbook-run.sh b/tests/installed/playbook-run.sh index 3136899f..37572bd4 100755 --- a/tests/installed/playbook-run.sh +++ b/tests/installed/playbook-run.sh @@ -32,4 +32,6 @@ rpm -q standard-test-roles export ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY:-$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory)} ls -al /dev/kvm +# Sadly having this on makes the reboot playbook break +export ANSIBLE_SSH_ARGS='-o ControlMaster=no' exec ansible-playbook --tags=atomic "$@" diff --git a/tests/installed/tasks/reboot.yml b/tests/installed/tasks/reboot.yml index fd077102..a5d0cbec 100644 --- a/tests/installed/tasks/reboot.yml +++ b/tests/installed/tasks/reboot.yml @@ -33,34 +33,35 @@ command: cat /proc/sys/kernel/random/boot_id register: orig_bootid +# Stop sshd (thus preventing new connections) and kill our current user's +# connection so that we can't race to get back in to the system while it's +# shutting down - name: restart hosts when: (not skip_shutdown is defined) or (not skip_shutdown) - shell: sleep 3 && shutdown -r now + shell: | + systemctl stop sshd + systemd-run --on-active=5 systemctl reboot async: 1 poll: 0 ignore_errors: true -# NB: The following tasks use local actions, so we need to explicitly ensure -# that they don't use sudo, which may require a password, and is not necessary -# anyway. +# NB: The wait_for is executed locally and doesn't require privs, so avoid sudo +- debug: + msg: "Waiting for reboot: {{ ansible_date_time.iso8601 }}" +- wait_for_connection: + delay: 5 + timeout: 120 + search_regex: "OpenSSH" +- debug: + msg: "SSH port is up {{ ansible_date_time.iso8601 }}" -- name: wait for hosts to come back up - local_action: - wait_for host={{ real_ansible_host }} - port={{ real_ansible_port | default('22') }} - state=started - delay=30 - timeout={{ timeout }} - search_regex="OpenSSH" - become: false - -# I'm not sure the retries are even necessary, but I'm keeping them in -- name: Wait until bootid changes +- name: Assert that the bootid changed command: cat /proc/sys/kernel/random/boot_id register: new_bootid until: new_bootid.stdout != orig_bootid.stdout - retries: 6 - delay: 10 +- assert: + that: + - new_bootid.stdout != orig_bootid.stdout # provide an empty iterator when a list is not provided # http://docs.ansible.com/ansible/playbooks_conditionals.html#loops-and-conditionals From 6d01d82b9b28d93364ada3cf7c199f08a4291fab Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 18 Apr 2018 15:57:59 -0400 Subject: [PATCH 41/80] tests/installed: increase async retries to 500 It seems like 240 retries is just not long enough for all the non-destructive tests running in parallel to finish. Let's crank that up to 500 retries. Closes: #1548 Approved by: cgwalters --- tests/installed/execute_batch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installed/execute_batch.yml b/tests/installed/execute_batch.yml index 7fd8374b..fe2b5702 100644 --- a/tests/installed/execute_batch.yml +++ b/tests/installed/execute_batch.yml @@ -19,4 +19,4 @@ loop_var: "async_result_item" register: async_poll_results until: async_poll_results.finished - retries: 240 + retries: 500 From 41b97e9c1267508249e6655ae9134e6fd49d2203 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Apr 2018 18:36:19 +0000 Subject: [PATCH 42/80] fsck: Only print "marking commit partial" once Let's only print if the commit isn't already partial; this addresses a spam of "marking commit partial" from fsck. Closes: #1548 Approved by: cgwalters --- src/ostree/ot-builtin-fsck.c | 11 +++++++++-- tests/installed/nondestructive/itest-pull.sh | 5 +++-- tests/test-corruption.sh | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 8a44b619..81124d3f 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -99,9 +99,16 @@ fsck_one_object (OstreeRepo *repo, for (i = 0; parent_commits[i] != NULL; i++) { const char *parent_commit = parent_commits[i]; - g_printerr ("Marking commit %s as partial\n", parent_commit); - if (!ostree_repo_mark_commit_partial (repo, parent_commit, TRUE, error)) + OstreeRepoCommitState state; + if (!ostree_repo_load_commit (repo, parent_commit, NULL, + &state, error)) return FALSE; + if ((state & OSTREE_REPO_COMMIT_STATE_PARTIAL) == 0) + { + g_printerr ("Marking commit as partial: %s\n", parent_commit); + if (!ostree_repo_mark_commit_partial (repo, parent_commit, TRUE, error)) + return FALSE; + } } } } diff --git a/tests/installed/nondestructive/itest-pull.sh b/tests/installed/nondestructive/itest-pull.sh index b3a52a27..0cf0ec75 100755 --- a/tests/installed/nondestructive/itest-pull.sh +++ b/tests/installed/nondestructive/itest-pull.sh @@ -35,11 +35,12 @@ echo "ok pull" # fsck marks commits partial # https://github.com/ostreedev/ostree/pull/1533 for d in $(find bare-repo/objects/ -maxdepth 1 -type d); do - (find ${d} -name '*.file' || true) | head -20 | xargs rm -vf + (find ${d} -name '*.file' || true) | head -20 | xargs rm -f done -if ostree --repo=bare-repo fsck; then +if ostree --repo=bare-repo fsck |& tee fsck.txt; then fatal "fsck unexpectedly succeeded" fi +assert_streq $(grep -cE -e 'Marking commit as partial' fsck.txt) "1" ostree --repo=bare-repo pull origin ${host_nonremoteref} # Don't need a full fsck here ostree --repo=bare-repo ls origin:${host_nonremoteref} >/dev/null diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index 3b4a649e..f26ed2d2 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -105,7 +105,7 @@ if $OSTREE fsck -q 2>err.txt; then assert_not_reached "fsck unexpectedly succeeded" fi assert_file_has_content_literal err.txt "Object missing:" -assert_file_has_content_literal err.txt "Marking commit $rev as partial" +assert_file_has_content_literal err.txt "Marking commit as partial: $rev" assert_has_file repo/state/${rev}.commitpartial echo "ok missing file" @@ -125,7 +125,7 @@ if $OSTREE fsck -q --delete 2>err.txt; then assert_not_reached "fsck unexpectedly succeeded" fi assert_file_has_content_literal err.txt "Corrupted file object;" -assert_file_has_content_literal err.txt "Marking commit $rev as partial" +assert_file_has_content_literal err.txt "Marking commit as partial: $rev" assert_has_file repo/state/${rev}.commitpartial echo "ok corrupt file" From 76f3e603f96817f826dbd7c93b0d0c28bfe1c0b7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 20 Apr 2018 14:16:15 -0400 Subject: [PATCH 43/80] tests: Lower retry timeout to 5s No need to poll every second, there's going to be some latency here and we want to avoid the overhead of polling. Closes: #1548 Approved by: cgwalters --- tests/installed/execute_batch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/installed/execute_batch.yml b/tests/installed/execute_batch.yml index fe2b5702..e4a11bb5 100644 --- a/tests/installed/execute_batch.yml +++ b/tests/installed/execute_batch.yml @@ -20,3 +20,4 @@ register: async_poll_results until: async_poll_results.finished retries: 500 + retry_pause: 5 From d4282720c78996dbb20e0bcfa5f730b28ad0b70b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 20 Apr 2018 14:17:20 -0400 Subject: [PATCH 44/80] tests: Disable itest-pull.sh since it is too slow Hopefully we'll fix this soon. Closes: #1548 Approved by: cgwalters --- tests/installed/nondestructive/itest-pull.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/installed/nondestructive/itest-pull.sh b/tests/installed/nondestructive/itest-pull.sh index 0cf0ec75..fc2047ed 100755 --- a/tests/installed/nondestructive/itest-pull.sh +++ b/tests/installed/nondestructive/itest-pull.sh @@ -4,6 +4,9 @@ set -xeuo pipefail +# FIXME: https://github.com/ostreedev/ostree/pull/1548 +exit 0 + dn=$(dirname $0) . ${dn}/../libinsttest.sh date @@ -37,11 +40,13 @@ echo "ok pull" for d in $(find bare-repo/objects/ -maxdepth 1 -type d); do (find ${d} -name '*.file' || true) | head -20 | xargs rm -f done +date if ostree --repo=bare-repo fsck |& tee fsck.txt; then fatal "fsck unexpectedly succeeded" fi +date assert_streq $(grep -cE -e 'Marking commit as partial' fsck.txt) "1" -ostree --repo=bare-repo pull origin ${host_nonremoteref} +log_timestamps ostree --repo=bare-repo pull origin ${host_nonremoteref} # Don't need a full fsck here ostree --repo=bare-repo ls origin:${host_nonremoteref} >/dev/null From 8073905ca5a772e9f9deb287b03c4114e8729b64 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 23 Apr 2018 09:23:17 -0400 Subject: [PATCH 45/80] ci: mark some ci testsuites as required again Follow up to #1536; we are now running all the testsuites on merges, but we weren't actually blocking on their success! Closes: #1552 Approved by: cgwalters --- .papr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.papr.yml b/.papr.yml index a1a6741f..7d76d247 100644 --- a/.papr.yml +++ b/.papr.yml @@ -32,6 +32,7 @@ branches: - master - auto - try +required: true container: image: registry.fedoraproject.org/fedora:27 context: f27-primary From c0c9cfdb9d33da671a47d32ba67a0915604152c0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 19 Apr 2018 10:16:35 -0400 Subject: [PATCH 46/80] ci: Temporary libgcrypt workaround for older kernels I'm trying to get ostree tests to pass in OpenShift as part of our CI move but I've been seeing lots of failures related to GPG tests. It finally turned out to be because libgcrypt doesn't behave well on older kernels that don't have `getrandom()` (the cluster is running on RHEL7). Thankfully, there's a new build with a fix for this. Pull that in manually until it gets into stable. For more information, see: https://bugzilla.redhat.com/show_bug.cgi?id=1542453 Closes: #1547 Approved by: cgwalters --- ci/libpaprci/libbuild.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh index 9ad417b2..62dc2b29 100644 --- a/ci/libpaprci/libbuild.sh +++ b/ci/libpaprci/libbuild.sh @@ -66,6 +66,10 @@ pkg_builddep() { if rpm -q gpgme | grep -q gpgme-1.9.0-6.fc27; then dnf install -y https://kojipkgs.fedoraproject.org//packages/gpgme/1.10.0/4.fc27/x86_64/{gpgme{,-devel},python{2,3}-gpg}-1.10.0-4.fc27.x86_64.rpm fi + # https://bugzilla.redhat.com/show_bug.cgi?id=1542453 + if rpm -q libgcrypt | grep -q libgcrypt-1.8.2-1.fc27; then + dnf install -y https://kojipkgs.fedoraproject.org//packages/libgcrypt/1.8.2/2.fc27/x86_64/libgcrypt-1.8.2-2.fc27.x86_64.rpm + fi else yum-builddep -y "$@" fi From d56058bc9a539ce344c669d26faaafe6c3071def Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 19 Apr 2018 09:58:49 -0400 Subject: [PATCH 47/80] tests/installed: Prefer python3 over python2 In a world progressively unapproving of python2, let's be a bit smarter and support testing on platforms that only have python3 installed. Closes: #1546 Approved by: cgwalters --- tests/installed/libinsttest.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh index 4968adc0..1c192b8e 100644 --- a/tests/installed/libinsttest.sh +++ b/tests/installed/libinsttest.sh @@ -37,6 +37,16 @@ prepare_tmpdir() { cd ${test_tmpdir} } +if test -x /usr/bin/python3; then + export PYTHON=/usr/bin/python3 + export PYTHONHTTPSERVER=http.server +elif test -x /usr/bin/python; then + export PYTHON=/usr/bin/python + export PYTHONHTTPSERVER=SimpleHTTPServer +else + fatal "no python found" +fi + # This is copied from flatpak/flatpak/tests/test-webserver.sh run_tmp_webserver() { dir=$1 @@ -44,22 +54,22 @@ run_tmp_webserver() { test -n ${test_tmpdir} cd ${dir} - env PYTHONUNBUFFERED=1 setsid python -m SimpleHTTPServer 0 &>${test_tmpdir}/httpd-output & + env PYTHONUNBUFFERED=1 setsid $PYTHON -m $PYTHONHTTPSERVER 0 &>${test_tmpdir}/httpd-output & cd - child_pid=$! - for x in $(seq 50); do + for x in $(seq 10); do # Snapshot the output cp ${test_tmpdir}/httpd-output{,.tmp} # If it's non-empty, see whether it matches our regexp - if test -s ${test_tmpdir}/httpd-output.tmp; then - sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port + if test -s ${test_tmpdir}/httpd-output.tmp; then # py3's http.server prints the http:// address also + sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\)\( (http://[^)]*)\)\? \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then # If so, we've successfully extracted the port break fi fi - sleep 0.1 + sleep 1 done port=$(cat ${test_tmpdir}/httpd-port) echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address @@ -75,16 +85,6 @@ fi # We need to be root assert_streq $(id -u) 0 -PYTHON= -for py in /usr/bin/python3 /usr/bin/python; do - if ! test -x ${py}; then continue; fi - export PYTHON=${py} - break -done -if test -z "${PYTHON}"; then - fatal "no python found" -fi - rpmostree_query_json() { query=$1 rpm-ostree status --json | $PYTHON -c 'import json,sys; v=json.load(sys.stdin); print(v'${query}')' From 8cfef187c5e80eac90a61fbbe98a0c7dd3ff9930 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 19 Apr 2018 09:58:51 -0400 Subject: [PATCH 48/80] tests: Port the last python2 script to python3 This is not used by any test, nor is it packaged. Though let's just port it over to py3 to certify our codebase completely py2-free. I've manually checked that the script is still functional. Closes: #1546 Approved by: cgwalters --- tests/grub2-entries-crosscheck.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/grub2-entries-crosscheck.py b/tests/grub2-entries-crosscheck.py index d68394d3..f2a68115 100644 --- a/tests/grub2-entries-crosscheck.py +++ b/tests/grub2-entries-crosscheck.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright (C) 2015 Red Hat # @@ -19,6 +19,7 @@ import os import sys +import functools if len(sys.argv) == 1: loaderpath = '/boot/loader/entries' @@ -40,7 +41,7 @@ def get_ostree_option(optionstring): if o.startswith('ostree='): return o[8:] raise ValueError('ostree= not found') - + entries = [] grub2_entries = [] @@ -59,7 +60,7 @@ for fname in os.listdir(loaderpath): v = line[s+1:] entry[k] = v entries.append(entry) - entries.sort(compare_entries_descending) + entries.sort(key=functools.cmp_to_key(compare_entries_descending)) # Parse GRUB2 config with open(grub2path) as f: @@ -68,7 +69,7 @@ with open(grub2path) as f: for line in f: if line.startswith('### BEGIN /etc/grub.d/15_ostree ###'): in_ostree_config = True - elif line.startswith('### END /etc/grub.d/15_ostree ###'): + elif line.startswith('### END /etc/grub.d/15_ostree ###'): in_ostree_config = False if grub2_entry is not None: grub2_entries.append(grub2_entry) From 01ea9a12e4713bf81d1ef0228d0813746c46bbb8 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 23 Apr 2018 13:36:38 -0400 Subject: [PATCH 49/80] ci: drop BDB1539 workaround This shouldn't be an issue anymore. Closes: #1553 Approved by: cgwalters --- ci/libpaprci/libbuild.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh index 62dc2b29..6ce7dcec 100644 --- a/ci/libpaprci/libbuild.sh +++ b/ci/libpaprci/libbuild.sh @@ -6,18 +6,7 @@ OS_ID=$(. /etc/os-release; echo $ID) OS_VERSION_ID=$(. /etc/os-release; echo $VERSION_ID) pkg_upgrade() { - # https://bugzilla.redhat.com/show_bug.cgi?id=1483553 - local ecode=0 - yum -y distro-sync 2>err.txt || ecode=$? - if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then - rpm --rebuilddb - yum -y distro-sync - else - if test ${ecode} '!=' 0; then - cat err.txt - exit ${ecode} - fi - fi + yum -y distro-sync } make() { From ab8d694361a91d0a181149c7ecb1ad69dd2a9353 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 24 Apr 2018 12:50:11 -0400 Subject: [PATCH 50/80] lib/deploy: Factor out function to get deployment kargs No functional change. Prep for next commit. Closes: #1556 Approved by: cgwalters --- src/libostree/ostree-sysroot-deploy.c | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 3843a89e..6e5d19c0 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1896,6 +1896,17 @@ assign_bootserials (GPtrArray *deployments) } } +static char* +get_deployment_nonostree_kargs (OstreeDeployment *deployment) +{ + /* pick up kernel arguments but filter out ostree= */ + OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (deployment); + const char *boot_options = ostree_bootconfig_parser_get (bootconfig, "options"); + g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_from_string (boot_options); + _ostree_kernel_args_replace (kargs, "ostree"); + return _ostree_kernel_args_to_string (kargs); +} + /* OSTree implements a special optimization where we want to avoid touching * the bootloader configuration if the kernel layout hasn't changed. This is * handled by the ostree= kernel argument referring to a "bootlink". But @@ -1909,29 +1920,18 @@ static gboolean deployment_bootconfigs_equal (OstreeDeployment *a, OstreeDeployment *b) { + /* same kernel & initramfs? */ const char *a_bootcsum = ostree_deployment_get_bootcsum (a); const char *b_bootcsum = ostree_deployment_get_bootcsum (b); - if (strcmp (a_bootcsum, b_bootcsum) != 0) return FALSE; - { - /* We checksum the kernel arguments *except* ostree= */ - OstreeBootconfigParser *a_bootconfig = ostree_deployment_get_bootconfig (a); - const char *a_boot_options = ostree_bootconfig_parser_get (a_bootconfig, "options"); - g_autoptr(OstreeKernelArgs) a_kargs = _ostree_kernel_args_from_string (a_boot_options); - _ostree_kernel_args_replace (a_kargs, "ostree"); - g_autofree char *a_boot_options_without_ostree = _ostree_kernel_args_to_string (a_kargs); + /* same kargs? */ + g_autofree char *a_boot_options_without_ostree = get_deployment_nonostree_kargs (a); + g_autofree char *b_boot_options_without_ostree = get_deployment_nonostree_kargs (b); + if (strcmp (a_boot_options_without_ostree, b_boot_options_without_ostree) != 0) + return FALSE; - OstreeBootconfigParser *b_bootconfig = ostree_deployment_get_bootconfig (b); - const char *b_boot_options = ostree_bootconfig_parser_get (b_bootconfig, "options"); - g_autoptr(OstreeKernelArgs) b_kargs = _ostree_kernel_args_from_string (b_boot_options); - _ostree_kernel_args_replace (b_kargs, "ostree"); - g_autofree char *b_boot_options_without_ostree = _ostree_kernel_args_to_string (b_kargs); - - if (strcmp (a_boot_options_without_ostree, b_boot_options_without_ostree) != 0) - return FALSE; - } return TRUE; } From dc4aa346a360d4fd30496f701d61a438039ec8af Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 24 Apr 2018 12:51:38 -0400 Subject: [PATCH 51/80] lib/deploy: Also compare deployment csum versions When comparing deployments to determine whether we need a new bootversion, we should also check whether the commit "version" metadata is the same. Otherwise, we may end up with the a bootconfig whose `title` includes a version that doesn't match the one from the deployment checksum. Closes: https://github.com/projectatomic/rpm-ostree/issues/1343 Closes: #1556 Approved by: cgwalters --- src/libostree/ostree-sysroot-deploy.c | 28 +++++++++++++++++++++++++-- tests/libtest.sh | 2 +- tests/test-admin-deploy-2.sh | 15 +++++++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 6e5d19c0..a3f00b4b 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1907,6 +1907,23 @@ get_deployment_nonostree_kargs (OstreeDeployment *deployment) return _ostree_kernel_args_to_string (kargs); } +static char* +get_deployment_ostree_version (OstreeRepo *repo, + OstreeDeployment *deployment) +{ + const char *csum = ostree_deployment_get_csum (deployment); + + g_autofree char *version = NULL; + g_autoptr(GVariant) variant = NULL; + if (ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, csum, &variant, NULL)) + { + g_autoptr(GVariant) metadata = g_variant_get_child_value (variant, 0); + g_variant_lookup (metadata, OSTREE_COMMIT_META_KEY_VERSION, "s", &version); + } + + return g_steal_pointer (&version); +} + /* OSTree implements a special optimization where we want to avoid touching * the bootloader configuration if the kernel layout hasn't changed. This is * handled by the ostree= kernel argument referring to a "bootlink". But @@ -1917,7 +1934,8 @@ get_deployment_nonostree_kargs (OstreeDeployment *deployment) * bootloader perspective. */ static gboolean -deployment_bootconfigs_equal (OstreeDeployment *a, +deployment_bootconfigs_equal (OstreeRepo *repo, + OstreeDeployment *a, OstreeDeployment *b) { /* same kernel & initramfs? */ @@ -1932,6 +1950,11 @@ deployment_bootconfigs_equal (OstreeDeployment *a, if (strcmp (a_boot_options_without_ostree, b_boot_options_without_ostree) != 0) return FALSE; + /* same ostree version? this is just for the menutitle, we won't have to cp the kernel */ + g_autofree char *a_version = get_deployment_ostree_version (repo, a); + g_autofree char *b_version = get_deployment_ostree_version (repo, b); + if (g_strcmp0 (a_version, b_version) != 0) + return FALSE; return TRUE; } @@ -2191,13 +2214,14 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, else { gboolean is_noop = TRUE; + OstreeRepo *repo = ostree_sysroot_repo (self); for (guint i = 0; i < new_deployments->len; i++) { OstreeDeployment *cur_deploy = self->deployments->pdata[i]; if (ostree_deployment_is_staged (cur_deploy)) continue; OstreeDeployment *new_deploy = new_deployments->pdata[i]; - if (!deployment_bootconfigs_equal (cur_deploy, new_deploy)) + if (!deployment_bootconfigs_equal (repo, cur_deploy, new_deploy)) { requires_new_bootversion = TRUE; is_noop = FALSE; diff --git a/tests/libtest.sh b/tests/libtest.sh index 586bf9ef..f6b6eb2f 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -491,7 +491,7 @@ os_repository_new_commit () echo "content iteration ${content_iteration}" > usr/bin/content-iteration - version=$(date "+%Y%m%d.${content_iteration}") + export version=$(date "+%Y%m%d.${content_iteration}") ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" -b $branch -s "Build" cd ${test_tmpdir} diff --git a/tests/test-admin-deploy-2.sh b/tests/test-admin-deploy-2.sh index eab0a3d3..7e69ec88 100755 --- a/tests/test-admin-deploy-2.sh +++ b/tests/test-admin-deploy-2.sh @@ -26,7 +26,7 @@ set -euo pipefail # Exports OSTREE_SYSROOT so --sysroot not needed. setup_os_repository "archive" "syslinux" -echo "1..6" +echo "1..7" ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmaster/x86_64-runtime rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) @@ -64,6 +64,19 @@ assert_file_has_content sysroot/ostree/deploy/testos/deploy/${newrev}.0/etc/os-r echo "ok manual cleanup" +# Commit + upgrade twice, so that we'll rotate out the original deployment +os_repository_new_commit "1" +${CMD_PREFIX} ostree admin upgrade --os=testos +oldversion=${version} +# another commit with *same* bootcsum but *new* content +os_repository_new_commit "1" "2" +newversion=${version} +assert_file_has_content sysroot/boot/loader/entries/ostree-testos-0.conf ${oldversion} +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_file_has_content sysroot/boot/loader/entries/ostree-testos-0.conf ${newversion} + +echo "ok new version same bootcsum" + assert_n_pinned() { local n=$1 ${CMD_PREFIX} ostree admin status > status.txt From de4c7105f019f514a56f0ec6911b865b3f2d58a1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 27 Apr 2018 11:53:43 -0400 Subject: [PATCH 52/80] lib/deploy: Fix staged deployments with no kargs Testing out the staged API with rpm-ostree, ostree-prepare-root.service in the initramfs was failing. Turned out that was because we didn't have a `root=` kernel argument. Which was because we didn't have any kernel arguments at all except `ostree=`. That in turn was because we weren't loading the bootloader config from the merge deployment. The serialized deployment data holds the unique identity of (osname, checksum, deployserial) - look for the real merge deployment in our deployment list which has the bootloader arguments we need. This issue was entirely masked by the `ostree admin deploy` command which itself explicitly loads the merge deployment's kernel arguments in every case - it never passes the `NULL` default down. A followup patch will fix that. Closes: #1558 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index a3f00b4b..b573325c 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2759,10 +2759,24 @@ _ostree_sysroot_finalize_staged (OstreeSysroot *self, if (g_variant_lookup (self->staged_deployment_data, "merge-deployment", "@a{sv}", &merge_deployment_v)) { - merge_deployment = + g_autoptr(OstreeDeployment) merge_deployment_stub = _ostree_sysroot_deserialize_deployment_from_variant (merge_deployment_v, error); - if (!merge_deployment) + if (!merge_deployment_stub) return FALSE; + for (guint i = 0; i < self->deployments->len; i++) + { + OstreeDeployment *deployment = self->deployments->pdata[i]; + if (ostree_deployment_equal (deployment, merge_deployment_stub)) + { + merge_deployment = g_object_ref (deployment); + break; + } + } + + if (!merge_deployment) + return glnx_throw (error, "Failed to find merge deployment %s.%d for staged", + ostree_deployment_get_csum (merge_deployment_stub), + ostree_deployment_get_deployserial (merge_deployment_stub)); } g_autofree char **kargs = NULL; g_variant_lookup (self->staged_deployment_data, "kargs", "^a&s", &kargs); From 06d39efcb5ad52216a2a47b5eff71af82c0dae49 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 27 Apr 2018 11:58:50 -0400 Subject: [PATCH 53/80] bin/deploy: Avoid loading merge deployment kargs unless necessary The fact that `ostree admin deploy` always itself loaded the merge kargs masked a bug in the core. Let's change our tests to not pass any kernel arguments to ensure we cover this. The new logic in the CLI is a bit subtle, but if you read carefully is a lot clearer I believe. Basically we have one of a few "starting points" in the first section, which can then be further augmented. Closes: #1558 Approved by: jlebon --- src/ostree/ot-admin-builtin-deploy.c | 30 ++++++++++++------- tests/installed/destructive/staged-deploy.yml | 13 +++----- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index f6c0c161..0a1755fd 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -65,8 +65,6 @@ static GOptionEntry options[] = { gboolean ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { - g_autoptr(OstreeKernelArgs) kargs = NULL; - g_autoptr(GOptionContext) context = g_option_context_new ("REFSPEC"); @@ -129,36 +127,48 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat if (!ostree_sysroot_prepare_cleanup (sysroot, cancellable, error)) return glnx_prefix_error (error, "Performing initial cleanup"); - kargs = _ostree_kernel_args_new (); - - /* If they want the current kernel's args, they very likely don't - * want the ones from the merge. + /* Initial set of kernel arguments; the default is to use the merge + * deployment, unless --karg-none or --karg-proc-cmdline are specified. */ - if (opt_kernel_proc_cmdline) + g_autoptr(OstreeKernelArgs) kargs = NULL; + if (opt_kernel_arg_none) { + kargs = _ostree_kernel_args_new (); + } + else if (opt_kernel_proc_cmdline) + { + kargs = _ostree_kernel_args_new (); if (!_ostree_kernel_args_append_proc_cmdline (kargs, cancellable, error)) return FALSE; } - else if (merge_deployment && !opt_kernel_arg_none) + else if (merge_deployment && (opt_kernel_argv || opt_kernel_argv_append)) { OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (merge_deployment); g_auto(GStrv) previous_args = g_strsplit (ostree_bootconfig_parser_get (bootconfig, "options"), " ", -1); - + kargs = _ostree_kernel_args_new (); _ostree_kernel_args_append_argv (kargs, previous_args); } + /* Now replace/extend the above set. Note that if no options are specified, + * we should end up passing NULL as override_kernel_argv for + * ostree_sysroot_deploy_tree() so we get the defaults. + */ if (opt_kernel_argv) { + if (!kargs) + kargs = _ostree_kernel_args_new (); _ostree_kernel_args_replace_argv (kargs, opt_kernel_argv); } if (opt_kernel_argv_append) { + if (!kargs) + kargs = _ostree_kernel_args_new (); _ostree_kernel_args_append_argv (kargs, opt_kernel_argv_append); } g_autoptr(OstreeDeployment) new_deployment = NULL; - g_auto(GStrv) kargs_strv = _ostree_kernel_args_to_strv (kargs); + g_auto(GStrv) kargs_strv = kargs ? _ostree_kernel_args_to_strv (kargs) : NULL; if (opt_stage) { if (opt_retain_pending || opt_retain_rollback) diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index 3f4062b2..016fcd6a 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -4,7 +4,7 @@ - name: Write staged-deploy commit shell: | ostree --repo=/ostree/repo commit --parent="${commit}" -b staged-deploy --tree=ref="${commit}" --no-bindings - ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment environment: commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" @@ -13,8 +13,6 @@ shell: | # Assert that the previous boot had a journal entry for it journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' - # And that we have the new kernel argument - grep -q -e 'ostreetest=yes' /proc/cmdline # And there should not be a staged deployment test '!' -f /run/ostree/staged-deployment - name: Rollback @@ -22,14 +20,11 @@ - include_tasks: ../tasks/reboot.yml - shell: | rpm-ostree cleanup -rp -# And now we shouldn't have the kernel commandline entry -- name: Check we do not have new kernel cmdline entry - shell: grep -qv -e 'ostreetest=yes' /proc/cmdline # Ensure we can unstage - name: Write staged-deploy commit, then unstage shell: | - ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + ostree admin deploy --stage staged-deploy ostree admin status > status.txt grep -qFe '(staged)' status.txt test -f /run/ostree/staged-deployment @@ -43,10 +38,10 @@ # Staged should be overwritten by non-staged - name: Write staged-deploy commit, then unstage shell: | - ostree admin deploy --stage --karg-proc-cmdline --karg=ostreetest=yes staged-deploy + ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment ostree --repo=/ostree/repo commit --parent="${commit}" -b nonstaged-deploy --tree=ref="${commit}" --no-bindings - ostree admin deploy --karg-proc-cmdline --karg=ostreetest=yes nonstaged-deploy + ostree admin deploy nonstaged-deploy ostree admin status > status.txt grep -vqFe '(staged)' status.txt test '!' -f /run/ostree/staged-deployment From cd17e364ae34c161641e8fe973fd1c11702a0d81 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 27 Apr 2018 15:33:02 -0400 Subject: [PATCH 54/80] deploy: Return staged deployment Today rpm-ostree has some code to run a "sanitycheck" on a deployment. I had initially deleted that when adapting it to use the staging code, but I realized it should work fine; we just won't see the merged config, but that's OK. When I readded that code it started crashing because we didn't actually return the new deployment object. We'll gain some coverage here as I'll land the code to have rpm-ostree use staging, then bump the rpm-ostree tests here. Closes: #1559 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 1 + src/ostree/ot-admin-builtin-deploy.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index b573325c..4314520a 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2736,6 +2736,7 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, if (!ostree_sysroot_load (self, cancellable, error)) return FALSE; + ot_transfer_out_value (out_new_deployment, &deployment); return TRUE; } diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index 0a1755fd..38ec923f 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -178,12 +178,14 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat if (!ostree_sysroot_stage_tree (sysroot, opt_osname, revision, origin, merge_deployment, kargs_strv, &new_deployment, cancellable, error)) return FALSE; + g_assert (new_deployment); } else { if (!ostree_sysroot_deploy_tree (sysroot, opt_osname, revision, origin, merge_deployment, kargs_strv, &new_deployment, cancellable, error)) return FALSE; + g_assert (new_deployment); OstreeSysrootSimpleWriteDeploymentFlags flags = OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN; if (opt_retain) From 776ce5cc6afb77d971c7fa1fed86ba1badb7df21 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 23 Apr 2018 17:01:53 -0400 Subject: [PATCH 55/80] README.md: Fix docs link to COPYING Because `README.md` also gets rendered into the docs, we need to link directly to github. Closes: https://github.com/ostreedev/ostree/issues/1534 Closes: #1554 Approved by: jlebon --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bf9088bd..485bc26d 100644 --- a/README.md +++ b/README.md @@ -113,5 +113,8 @@ See [Contributing](CONTRIBUTING.md). Licensing ------- -The license for the *code* of libostree can be found in [COPYING](COPYING). +The licensing for the *code* of libostree can be canonically found in the individual files; +and the overall status in the [COPYING](https://github.com/ostreedev/ostree/blob/master/COPYING) +file in the source. Currently, that's LGPLv2+. + The license for the *documentation* of libostree is: `SPDX-License-Identifier: (CC-BY-SA-3.0 OR GFDL-1.3-or-later)` From 8c1542134cf8a1a00d669c80da45c15c22fc0522 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 23 Apr 2018 17:53:04 -0400 Subject: [PATCH 56/80] lib/repo: Enable locking by default, but drop external API The code has been sitting around for a while but since I disabled it by default, I doubt anyone is really using it or relying on it. This patch and turns on locking by default, and also drops the API which was only public in the experimental API builds. Conceptually these are two distinct things, and we may actually want to split up the patches. I don't think this will break anyone, but it's hard to say for sure. It's also going to be hard to find out until we actually release I suspect... But anyone who is broken should be able to add `locking=false` into their repo config. On the flip side Endless has been shipping with this enabled and it is reported to help. The reason to drop the APIs: I'm a bit concerned about the interactions over time between libostree's use of the API and any apps that start using it. For example, if an app specifies a SHARED lock in their code, then later internally we decide to temporarily grab an `EXCLUSIVE`, but the app had a second thread/process that was `EXCLUSIVE` already, and that process was waiting on the first bit of code, then we could deadlock. I can't think of a real world situation where this would happen yet though. We are likely to in the future have say `fsck` take an external lock, `checkout` grab a shared one, etc. Closes: #1555 Approved by: jlebon --- apidoc/ostree-experimental-sections.txt | 6 --- src/libostree/libostree-experimental.sym | 4 -- src/libostree/ostree-autocleanups.h | 1 - src/libostree/ostree-repo-commit.c | 8 ++-- src/libostree/ostree-repo-private.h | 32 +++++++------- src/libostree/ostree-repo-prune.c | 9 ++-- src/libostree/ostree-repo.c | 53 +++++++++++------------- src/libostree/ostree-repo.h | 42 ------------------- tests/test-concurrency.py | 1 - 9 files changed, 48 insertions(+), 108 deletions(-) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 0d168406..60daaca5 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -90,12 +90,6 @@ ostree_repo_finder_override_get_type
ostree-misc-experimental -OstreeRepoLockType -ostree_repo_lock_push -ostree_repo_lock_pop -OstreeRepoAutoLock -ostree_repo_auto_lock_push -ostree_repo_auto_lock_cleanup ostree_repo_get_collection_id ostree_repo_set_collection_id ostree_validate_collection_id diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 3f3454f3..b83ad1b0 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -94,8 +94,4 @@ LIBOSTREE_2017.14_EXPERIMENTAL { global: ostree_remote_get_type; ostree_remote_get_url; - ostree_repo_auto_lock_cleanup; - ostree_repo_auto_lock_push; - ostree_repo_lock_pop; - ostree_repo_lock_push; } LIBOSTREE_2017.13_EXPERIMENTAL; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index 75b498fc..504954e0 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -61,7 +61,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref) G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear) #ifdef OSTREE_ENABLE_EXPERIMENTAL_API -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index c171b3da..6eb645be 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1567,8 +1567,8 @@ ostree_repo_prepare_transaction (OstreeRepo *self, memset (&self->txn.stats, 0, sizeof (OstreeRepoTransactionStats)); - self->txn_locked = ostree_repo_lock_push (self, OSTREE_REPO_LOCK_SHARED, - cancellable, error); + self->txn_locked = _ostree_repo_lock_push (self, OSTREE_REPO_LOCK_SHARED, + cancellable, error); if (!self->txn_locked) return FALSE; @@ -2136,7 +2136,7 @@ ostree_repo_commit_transaction (OstreeRepo *self, if (self->txn_locked) { - if (!ostree_repo_lock_pop (self, cancellable, error)) + if (!_ostree_repo_lock_pop (self, cancellable, error)) return FALSE; self->txn_locked = FALSE; } @@ -2189,7 +2189,7 @@ ostree_repo_abort_transaction (OstreeRepo *self, if (self->txn_locked) { - if (!ostree_repo_lock_pop (self, cancellable, error)) + if (!_ostree_repo_lock_pop (self, cancellable, error)) return FALSE; self->txn_locked = FALSE; } diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 3078a9e2..77203638 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -435,34 +435,36 @@ _ostree_repo_get_remote_inherited (OstreeRepo *self, const char *name, GError **error); -#ifndef OSTREE_ENABLE_EXPERIMENTAL_API - -/* All the locking APIs below are duplicated in ostree-repo.h. Remove the ones - * here once it's no longer experimental. +/* Locking APIs are currently private. + * See https://github.com/ostreedev/ostree/pull/1555 */ - typedef enum { OSTREE_REPO_LOCK_SHARED, OSTREE_REPO_LOCK_EXCLUSIVE } OstreeRepoLockType; -gboolean ostree_repo_lock_push (OstreeRepo *self, +gboolean _ostree_repo_lock_push (OstreeRepo *self, OstreeRepoLockType lock_type, GCancellable *cancellable, GError **error); -gboolean ostree_repo_lock_pop (OstreeRepo *self, - GCancellable *cancellable, - GError **error); +gboolean _ostree_repo_lock_pop (OstreeRepo *self, + GCancellable *cancellable, + GError **error); typedef OstreeRepo OstreeRepoAutoLock; -OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo *self, - OstreeRepoLockType lock_type, - GCancellable *cancellable, - GError **error); -void ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock); -G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup) +OstreeRepoAutoLock * _ostree_repo_auto_lock_push (OstreeRepo *self, + OstreeRepoLockType lock_type, + GCancellable *cancellable, + GError **error); +void _ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock); +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, _ostree_repo_auto_lock_cleanup) +#ifndef OSTREE_ENABLE_EXPERIMENTAL_API + +/* These APIs are duplicated in the public headers when doing an + * experimental-API build. + */ const gchar * ostree_repo_get_collection_id (OstreeRepo *self); gboolean ostree_repo_set_collection_id (OstreeRepo *self, const gchar *collection_id, diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index f0c0a974..2ffd6948 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -201,8 +201,7 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, GError **error) { g_autoptr(OstreeRepoAutoLock) lock = - ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, - error); + _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); if (!lock) return FALSE; @@ -340,8 +339,7 @@ ostree_repo_prune (OstreeRepo *self, GError **error) { g_autoptr(OstreeRepoAutoLock) lock = - ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, - error); + _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); if (!lock) return FALSE; @@ -452,8 +450,7 @@ ostree_repo_prune_from_reachable (OstreeRepo *self, GError **error) { g_autoptr(OstreeRepoAutoLock) lock = - ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, - error); + _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); if (!lock) return FALSE; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 7d593f50..79006a6b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -436,7 +436,7 @@ pop_repo_lock (OstreeRepo *self, return TRUE; } -/** +/* * ostree_repo_lock_push: * @self: a #OstreeRepo * @lock_type: the type of lock to acquire @@ -462,13 +462,12 @@ pop_repo_lock (OstreeRepo *self, * %TRUE is returned. * * Returns: %TRUE on success, otherwise %FALSE with @error set - * Since: 2017.14 */ gboolean -ostree_repo_lock_push (OstreeRepo *self, - OstreeRepoLockType lock_type, - GCancellable *cancellable, - GError **error) +_ostree_repo_lock_push (OstreeRepo *self, + OstreeRepoLockType lock_type, + GCancellable *cancellable, + GError **error) { g_return_val_if_fail (self != NULL, FALSE); g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); @@ -531,8 +530,8 @@ ostree_repo_lock_push (OstreeRepo *self, } } -/** - * ostree_repo_lock_pop: +/* + * _ostree_repo_lock_pop: * @self: a #OstreeRepo * @cancellable: a #GCancellable * @error: a #GError @@ -553,12 +552,11 @@ ostree_repo_lock_push (OstreeRepo *self, * %TRUE is returned. * * Returns: %TRUE on success, otherwise %FALSE with @error set - * Since: 2017.14 */ gboolean -ostree_repo_lock_pop (OstreeRepo *self, - GCancellable *cancellable, - GError **error) +_ostree_repo_lock_pop (OstreeRepo *self, + GCancellable *cancellable, + GError **error) { g_return_val_if_fail (self != NULL, FALSE); g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); @@ -621,8 +619,8 @@ ostree_repo_lock_pop (OstreeRepo *self, } } -/** - * ostree_repo_auto_lock_push: (skip) +/* + * _ostree_repo_auto_lock_push: (skip) * @self: a #OstreeRepo * @lock_type: the type of lock to acquire * @cancellable: a #GCancellable @@ -636,37 +634,34 @@ ostree_repo_lock_pop (OstreeRepo *self, * * |[ * g_autoptr(OstreeRepoAutoLock) lock = NULL; - * lock = ostree_repo_auto_lock_push (repo, lock_type, cancellable, error); + * lock = _ostree_repo_auto_lock_push (repo, lock_type, cancellable, error); * if (!lock) * return FALSE; * ]| * * Returns: @self on success, otherwise %NULL with @error set - * Since: 2017.14 */ OstreeRepoAutoLock * -ostree_repo_auto_lock_push (OstreeRepo *self, - OstreeRepoLockType lock_type, - GCancellable *cancellable, - GError **error) +_ostree_repo_auto_lock_push (OstreeRepo *self, + OstreeRepoLockType lock_type, + GCancellable *cancellable, + GError **error) { - if (!ostree_repo_lock_push (self, lock_type, cancellable, error)) + if (!_ostree_repo_lock_push (self, lock_type, cancellable, error)) return NULL; return (OstreeRepoAutoLock *)self; } -/** - * ostree_repo_auto_lock_cleanup: (skip) +/* + * _ostree_repo_auto_lock_cleanup: (skip) * @lock: a #OstreeRepoAutoLock * * A cleanup handler for use with ostree_repo_auto_lock_push(). If @lock is * not %NULL, ostree_repo_lock_pop() will be called on it. If * ostree_repo_lock_pop() fails, a critical warning will be emitted. - * - * Since: 2017.14 */ void -ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock) +_ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock) { OstreeRepo *repo = lock; if (repo) @@ -674,7 +669,7 @@ ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock) g_autoptr(GError) error = NULL; int errsv = errno; - if (!ostree_repo_lock_pop (repo, NULL, &error)) + if (!_ostree_repo_lock_pop (repo, NULL, &error)) g_critical ("Cleanup repo lock failed: %s", error->message); errno = errsv; @@ -2742,10 +2737,10 @@ reload_core_config (OstreeRepo *self, self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10); } - /* Disable locking by default for now */ { gboolean locking; + /* Enabled by default in 2018.05 */ if (!ot_keyfile_get_boolean_with_default (self->config, "core", "locking", - FALSE, &locking, error)) + TRUE, &locking, error)) return FALSE; if (!locking) { diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 04b04416..8d3a7a6f 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -109,48 +109,6 @@ OstreeRepo * ostree_repo_create_at (int dfd, #ifdef OSTREE_ENABLE_EXPERIMENTAL_API -/** - * OstreeRepoLockType: - * @OSTREE_REPO_LOCK_SHARED: A shared lock - * @OSTREE_REPO_LOCK_EXCLUSIVE: An exclusive lock - * - * The type of repository lock to acquire. - * - * Since: 2017.14 - */ -typedef enum { - OSTREE_REPO_LOCK_SHARED, - OSTREE_REPO_LOCK_EXCLUSIVE -} OstreeRepoLockType; - -_OSTREE_PUBLIC -gboolean ostree_repo_lock_push (OstreeRepo *self, - OstreeRepoLockType lock_type, - GCancellable *cancellable, - GError **error); -_OSTREE_PUBLIC -gboolean ostree_repo_lock_pop (OstreeRepo *self, - GCancellable *cancellable, - GError **error); - -/** - * OstreeRepoAutoLock: (skip) - * - * This is simply an alias to #OstreeRepo used for automatic lock cleanup. - * See ostree_repo_auto_lock_push() for its intended usage. - * - * Since: 2017.14 - */ -typedef OstreeRepo OstreeRepoAutoLock; - -_OSTREE_PUBLIC -OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo *self, - OstreeRepoLockType lock_type, - GCancellable *cancellable, - GError **error); -_OSTREE_PUBLIC -void ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock); - _OSTREE_PUBLIC const gchar * ostree_repo_get_collection_id (OstreeRepo *self); _OSTREE_PUBLIC diff --git a/tests/test-concurrency.py b/tests/test-concurrency.py index 3ec3681c..e4ce21e9 100755 --- a/tests/test-concurrency.py +++ b/tests/test-concurrency.py @@ -44,7 +44,6 @@ subprocess.check_call(['ostree', '--repo=repo', 'init', '--mode=bare']) # and we don't need xattr coverage for this with open('repo/config', 'a') as f: f.write('disable-xattrs=true\n') - f.write('locking=true\n') def commit(v): tdir='tree{}'.format(v) From 54c5ab76a94bebc647cb0d80652f8d9980a2fa4d Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 30 Apr 2018 08:53:01 -0400 Subject: [PATCH 57/80] man/ostree: Document --repo option a bit more This new information is already mostly part of `ostree.repo(5)`, though let's put it in `ostree(1)` as well since that's where the switch is officially documented. Closes: #1560 Approved by: cgwalters --- man/ostree.xml | 16 ++++++++-------- src/ostree/ot-main.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/man/ostree.xml b/man/ostree.xml index 8a79cd04..f8d3e2fc 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -111,14 +111,14 @@ Boston, MA 02111-1307, USA. - For most commands, - when run as non-root, repository is - required. If - ostree is run as - root, it is assumed operations will be - performed on the - /sysroot/ostree/repo - repository. + For most commands, a repository is + required. If unspecified, the current + directory is used if it appears to be an + OSTree repository. If it isn't, either + the OSTREE_REPO + environment variable is used, or the + system repository located at + /sysroot/ostree/repo. diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 237427a2..148c8371 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -46,7 +46,7 @@ static GOptionEntry global_entries[] = { }; static GOptionEntry repo_entry[] = { - { "repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_repo, "Path to OSTree repository (defaults to /sysroot/ostree/repo)", "PATH" }, + { "repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_repo, "Path to OSTree repository (defaults to current directory or /sysroot/ostree/repo)", "PATH" }, { NULL } }; From a85f06952defd341f45a53a3cdfeef66d4003de0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 30 Apr 2018 09:42:11 -0400 Subject: [PATCH 58/80] man/ostree-init: Briefly describe various modes Closes: #1561 Closes: #1560 Approved by: cgwalters --- man/ostree-init.xml | 25 ++++++++++++++++++------- src/ostree/ot-builtin-init.c | 2 +- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/man/ostree-init.xml b/man/ostree-init.xml index b74648db..b2726321 100644 --- a/man/ostree-init.xml +++ b/man/ostree-init.xml @@ -69,13 +69,24 @@ Boston, MA 02111-1307, USA. ="MODE" - Initialize repository in given mode - (bare, bare-user, - archive). The default is - bare. Note that for - archive the repository configuration file - will actually have archive-z2, as that's the - historical name. + + Initialize repository in given mode + (bare, bare-user, + bare-user-only, archive). + The default is bare. Note that for + archive the repository configuration file + will actually have archive-z2, as that's + the historical name. + + See the manual for differences between these modes. + Briefly, bare mode stores files as they + are, so they can be directly hardlinked, + bare-user uses extended attributes to + store ownership and xattr information, allowing non-root + operations, bare-user-only does not store + ownership information, and archive stores + files compressed, to be served over the network. + diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c index ae0d0cc6..a4fb5c51 100644 --- a/src/ostree/ot-builtin-init.c +++ b/src/ostree/ot-builtin-init.c @@ -38,7 +38,7 @@ static char *opt_collection_id = NULL; */ static GOptionEntry options[] = { - { "mode", 0, 0, G_OPTION_ARG_STRING, &opt_mode, "Initialize repository in given mode (bare, archive)", NULL }, + { "mode", 0, 0, G_OPTION_ARG_STRING, &opt_mode, "Initialize repository in given mode (bare, bare-user, bare-user-only, archive)", NULL }, #ifdef OSTREE_ENABLE_EXPERIMENTAL_API { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, From f0708b3441623387dcc7a752c47eca6785d660ac Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 26 Apr 2018 18:29:49 +0100 Subject: [PATCH 59/80] build: Don't distribute Bison-generated parser in dist tarballs Signed-off-by: Simon McVittie Closes: #1563 Approved by: cgwalters --- Makefile-ostree.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index bdd51a72..91d8383d 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -52,6 +52,9 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-editor.c \ src/ostree/ot-editor.h \ src/ostree/parse-datetime.h \ + $(NULL) + +nodist_ostree_SOURCES = \ src/ostree/parse-datetime.c \ $(NULL) From 33d5b6f514c9fbe0b02764f8d999ed1531f23b51 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Apr 2018 11:29:47 -0400 Subject: [PATCH 60/80] bin/admin-cleanup: Port to decl-and-initialize style No functional changes. Closes: #1567 Approved by: jlebon --- src/ostree/ot-admin-builtin-cleanup.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/ostree/ot-admin-builtin-cleanup.c b/src/ostree/ot-admin-builtin-cleanup.c index a4753030..875d6fa3 100644 --- a/src/ostree/ot-admin-builtin-cleanup.c +++ b/src/ostree/ot-admin-builtin-cleanup.c @@ -37,21 +37,16 @@ static GOptionEntry options[] = { gboolean ot_admin_builtin_cleanup (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { - g_autoptr(GOptionContext) context = NULL; + g_autoptr(GOptionContext) context = g_option_context_new (""); + g_autoptr(OstreeSysroot) sysroot = NULL; - gboolean ret = FALSE; - - context = g_option_context_new (""); - if (!ostree_admin_option_context_parse (context, options, &argc, &argv, OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, invocation, &sysroot, cancellable, error)) - goto out; + return FALSE; if (!ostree_sysroot_cleanup (sysroot, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } From 11c12cc19a2d9e75038b9ab7e57213fa63316abe Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Apr 2018 14:41:30 -0400 Subject: [PATCH 61/80] deploy: Don't prune repo at finalization time by default Doing so can break rpm-ostree, which wants to own the cleanup process to ensure its baselayer refs are generated. Further, doing the cleanup at shutdown time adds latency. It's also going to be generally unnecessary as we expect repo pruning to have been done when writing the refs. Closes: #1567 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 4314520a..ea9dcb29 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2799,13 +2799,23 @@ _ostree_sysroot_finalize_staged (OstreeSysroot *self, staged->staged = FALSE; g_ptr_array_remove_index (self->deployments, 0); - /* TODO: Proxy across flags too? */ - OstreeSysrootSimpleWriteDeploymentFlags flags = 0; + /* TODO: Proxy across flags too? + * + * But note that we always use NO_CLEAN to avoid adding more latency at + * shutdown, and also because e.g. rpm-ostree wants to own the cleanup + * process. + */ + OstreeSysrootSimpleWriteDeploymentFlags flags = + OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN; if (!ostree_sysroot_simple_write_deployment (self, ostree_deployment_get_osname (staged), staged, merge_deployment, flags, cancellable, error)) return FALSE; + /* Do the basic cleanup that may impact /boot, but not the repo pruning */ + if (!ostree_sysroot_prepare_cleanup (self, cancellable, error)) + return FALSE; + return TRUE; } From 36ac768fb77c84046c61b355c965ff1b6b104ed3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 30 Apr 2018 09:46:29 -0400 Subject: [PATCH 62/80] repo: Add checksum to error message opening unreadable object This would have debugged trying to do a pull as non-root from a sysroot repository. See for example: https://github.com/ostreedev/ostree/issues/1562 Closes: #1564 Approved by: jlebon --- src/libostree/ostree-repo.c | 5 ++++- .../nondestructive/itest-bare-unit.sh | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 79006a6b..972f05ae 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3545,6 +3545,9 @@ _ostree_repo_load_file_bare (OstreeRepo *self, return FALSE; } + const char *errprefix = glnx_strjoina ("Opening content object ", checksum); + GLNX_AUTO_PREFIX_ERROR (errprefix, error); + struct stat stbuf; glnx_autofd int fd = -1; g_autofree char *ret_symlink = NULL; @@ -3585,7 +3588,7 @@ _ostree_repo_load_file_bare (OstreeRepo *self, } if (!(S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode))) - return glnx_throw (error, "Not a regular file or symlink: %s", loose_path_buf); + return glnx_throw (error, "Not a regular file or symlink"); /* In the non-bare-user case, gather symlink info if requested */ if (self->mode != OSTREE_REPO_MODE_BARE_USER diff --git a/tests/installed/nondestructive/itest-bare-unit.sh b/tests/installed/nondestructive/itest-bare-unit.sh index e3312608..c973ee7c 100755 --- a/tests/installed/nondestructive/itest-bare-unit.sh +++ b/tests/installed/nondestructive/itest-bare-unit.sh @@ -21,3 +21,21 @@ trap _tmpdir_cleanup EXIT /usr/libexec/installed-tests/libostree/test-basic.sh /usr/libexec/installed-tests/libostree/test-basic-c date + +# Test error message when opening a non-world-readable object +# https://github.com/ostreedev/ostree/issues/1562 +rm repo files -rf +chmod a+rx . +ostree --repo=repo init --mode=bare +mkdir files +touch files/unreadable +chmod 0 files/unreadable +ostree --repo=repo commit -b testbranch --tree=dir=files +# We should be able to read as root due to CAP_DAC_OVERRIDE +ostree --repo=repo cat testbranch /unreadable >/dev/null +if setpriv --reuid bin --regid bin --clear-groups ostree --repo=repo cat testbranch /unreadable 2>err.txt; then + fatal "Listed unreadable object as non-root" +fi +assert_file_has_content err.txt "Opening content object.*openat: Permission denied" + +date From f5f8e2c540ebfd89d34bda362d40af216ce6ba06 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 1 May 2018 20:29:55 +0000 Subject: [PATCH 63/80] lib/sysroot: Maintain one variable for "root is ostree booted" This was pointed out in a previous PR review; we don't have a need for the separate variables. Prep for adding an API for this. Closes: #1568 Approved by: jlebon --- src/libostree/ostree-sysroot-private.h | 3 +-- src/libostree/ostree-sysroot.c | 18 +++++++----------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index a2f3b869..16cca5a1 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -50,8 +50,7 @@ struct OstreeSysroot { GLnxLockFile lock; gboolean loaded; - gboolean ostree_booted; - gboolean root_is_sysroot; /* TRUE if sysroot_fd is pointed to rootfs "/" */ + gboolean root_is_ostree_booted; /* TRUE if sysroot is / and we are booted via ostree */ /* The device/inode for /, used to detect booted deployment */ dev_t root_device; ino_t root_inode; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index f59f1508..707b161f 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -636,10 +636,8 @@ parse_deployment (OstreeSysroot *self, return FALSE; /* See if this is the booted deployment */ - const gboolean root_is_ostree_booted = - (self->ostree_booted && self->root_is_sysroot); const gboolean looking_for_booted_deployment = - (root_is_ostree_booted && !self->booted_deployment); + (self->root_is_ostree_booted && !self->booted_deployment); gboolean is_booted_deployment = FALSE; if (looking_for_booted_deployment) { @@ -802,9 +800,7 @@ _ostree_sysroot_reload_staged (OstreeSysroot *self, GError **error) { GLNX_AUTO_PREFIX_ERROR ("Loading staged deployment", error); - const gboolean root_is_ostree_booted = - self->ostree_booted && self->root_is_sysroot; - if (!root_is_ostree_booted) + if (!self->root_is_ostree_booted) return TRUE; /* Note early return */ g_assert (self->booted_deployment); @@ -879,7 +875,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, { if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) return FALSE; - self->ostree_booted = (errno == 0); + const gboolean ostree_booted = (errno == 0); { struct stat root_stbuf; if (!glnx_fstatat (AT_FDCWD, "/", &root_stbuf, 0, error)) @@ -892,9 +888,11 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!glnx_fstat (self->sysroot_fd, &self_stbuf, error)) return FALSE; - self->root_is_sysroot = + const gboolean root_is_sysroot = (self->root_device == self_stbuf.st_dev && self->root_inode == self_stbuf.st_ino); + + self->root_is_ostree_booted = (ostree_booted && root_is_sysroot); } int bootversion = 0; @@ -947,9 +945,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, } } - const gboolean root_is_ostree_booted = - self->ostree_booted && self->root_is_sysroot; - if (root_is_ostree_booted && !self->booted_deployment) + if (self->root_is_ostree_booted && !self->booted_deployment) return glnx_throw (error, "Unexpected state: /run/ostree-booted found and in / sysroot but not in a booted deployment"); if (!_ostree_sysroot_reload_staged (self, error)) From 25ba8db987094b141647b2d2e721095b61fac5c1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 1 May 2018 16:36:03 -0400 Subject: [PATCH 64/80] lib/deploy: Throw an error if trying to stage when not ostree-booted There's no reason to do this. I didn't actually hit this problem, but it's a corner case that just occurred to me while working on the code. I think callers should be adapted to skip trying to use staging if there's no booted deployment. Closes: #1568 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 4 ++++ tests/admin-test.sh | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index ea9dcb29..6f6181d7 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2658,6 +2658,10 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (self); + if (booted_deployment == NULL) + return glnx_throw (error, "Cannot stage a deployment when not currently booted into an OSTree system"); + /* This is a bit of a hack. When adding a new service we have to end up getting * into the presets for downstream distros; see e.g. https://src.fedoraproject.org/rpms/ostree/pull-request/7 * diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 6e3fd782..dd87cd11 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((23 + ${extra_admin_tests:-0}))" +echo "1..$((24 + ${extra_admin_tests:-0}))" function validate_bootloader() { cd ${test_tmpdir}; @@ -78,6 +78,12 @@ assert_ostree_deployment_refs 1/1/0 ${CMD_PREFIX} ostree admin status echo "ok layout" +if ${CMD_PREFIX} ostree admin deploy --stage --os=testos testos:testos/buildmaster/x86_64-runtime 2>err.txt; then + fatal "staged when not booted" +fi +assert_file_has_content_literal err.txt "Cannot stage a deployment when not currently booted into an OSTree system" +echo "ok staging does not work when not booted" + orig_mtime=$(stat -c '%.Y' sysroot/ostree/deploy) ${CMD_PREFIX} ostree admin deploy --os=testos testos:testos/buildmaster/x86_64-runtime new_mtime=$(stat -c '%.Y' sysroot/ostree/deploy) From 5337ba51b2e46d5a5bb3d5c627bd724c77d90408 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 2 May 2018 14:33:32 +0000 Subject: [PATCH 65/80] lib/deploy: Do post-ops when removing staged commit These are further fixes based on running more of the rpm-ostree test suite. When dropping the staged deployment, we do need to do the "post operations" such as bumping the sysroot mtime, so that clients know something changed. We also need to regenerate the deployment refs. And of course do a sysroot reload. Also, add a "base cleanup" after creating a staged deployment which also regenerates the refs. Closes: #1570 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 51 +++++++++++++++---- tests/installed/destructive/staged-deploy.yml | 11 ++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 6f6181d7..b21be0e9 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2143,6 +2143,25 @@ write_deployments_bootswap (OstreeSysroot *self, return TRUE; } +/* Actions taken after writing deployments is complete */ +static gboolean +write_deployments_finish (OstreeSysroot *self, + GCancellable *cancellable, + GError **error) +{ + if (!_ostree_sysroot_bump_mtime (self, error)) + return FALSE; + + /* Now reload from disk */ + if (!ostree_sysroot_load (self, cancellable, error)) + return glnx_prefix_error (error, "Reloading deployments after commit"); + + if (!cleanup_legacy_current_symlinks (self, cancellable, error)) + return FALSE; + + return TRUE; +} + /** * ostree_sysroot_write_deployments_with_options: * @self: Sysroot @@ -2173,6 +2192,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, * now, which is mostly what this function is concerned with. * In the future we though should probably adapt things to keep it. */ + gboolean removed_staged = FALSE; if (self->staged_deployment) { if (!glnx_unlinkat (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED, 0, error)) @@ -2183,6 +2203,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, g_assert (self->staged_deployment == self->deployments->pdata[0]); g_ptr_array_remove_index (self->deployments, 0); + removed_staged = TRUE; } /* First new deployment; we'll see if it's staged */ OstreeDeployment *first_new = @@ -2231,10 +2252,22 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, is_noop = FALSE; } - /* Silently do nothing if we're passed the same set of deployments */ + /* If we're passed the same set of deployments, we don't need + * to drop into the rest of this function which deals with + * changing the bootloader config. + */ if (is_noop) { g_assert (!requires_new_bootversion); + /* However, if we dropped the staged deployment, we still + * need to do finalization steps such as regenerating + * the refs and bumping the mtime. + */ + if (removed_staged) + { + if (!write_deployments_finish (self, cancellable, error)) + return FALSE; + } return TRUE; } } @@ -2338,14 +2371,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, _ostree_sysroot_emit_journal_msg (self, msg); } - if (!_ostree_sysroot_bump_mtime (self, error)) - return FALSE; - - /* Now reload from disk */ - if (!ostree_sysroot_load (self, cancellable, error)) - return glnx_prefix_error (error, "Reloading deployments after commit"); - - if (!cleanup_legacy_current_symlinks (self, cancellable, error)) + if (!write_deployments_finish (self, cancellable, error)) return FALSE; /* And finally, cleanup of any leftover data. @@ -2739,6 +2765,13 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, return FALSE; if (!ostree_sysroot_load (self, cancellable, error)) return FALSE; + /* Like deploy, we do a prepare cleanup; among other things, this ensures + * that a ref will be written for the staged tree. See also + * https://github.com/ostreedev/ostree/pull/1566 though which + * adds an ostree_sysroot_cleanup_prune() API. + */ + if (!ostree_sysroot_prepare_cleanup (self, cancellable, error)) + return FALSE; ot_transfer_out_value (out_new_deployment, &deployment); return TRUE; diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index 016fcd6a..f946e139 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -4,8 +4,19 @@ - name: Write staged-deploy commit shell: | ostree --repo=/ostree/repo commit --parent="${commit}" -b staged-deploy --tree=ref="${commit}" --no-bindings + newcommit=$(ostree rev-parse staged-deploy) + orig_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) ostree admin deploy --stage staged-deploy + new_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) + assert_not_streq "${orig_mtime}" "${new_mtime}" test -f /run/ostree/staged-deployment + ostree refs | grep -E -e '^ostree/' | while read ref; do + if test "$(ostree rev-parse ${ref})" = "${newcommit}"; then + touch deployment-ref-found + fi + done + test -f deployment-ref-found + rm deployment-ref-found environment: commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" - include_tasks: ../tasks/reboot.yml From 9f8e2b886235d0988326cbdcb270678458060b81 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 2 May 2018 12:17:20 -0400 Subject: [PATCH 66/80] lib: Use `Locking:` term in docs This is easier to `git grep` etc. versus ad-hoc English. Although we still have some English for the prepare_transaction/commit which acquire/release in separate phases. Closes: #1572 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 4 ++-- src/libostree/ostree-repo-prune.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 6eb645be..2ddcf498 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1551,10 +1551,9 @@ ostree_repo_scan_hardlinks (OstreeRepo *self, * on a single `OstreeRepo` instance as long as their lifetime is bounded by the * transaction. * + * Locking: Acquires a `shared` lock; release via commit or abort * Multithreading: This function is *not* MT safe; only one transaction can be * active at a time. - * - * This function takes a shared lock on the @self repository. */ gboolean ostree_repo_prepare_transaction (OstreeRepo *self, @@ -2080,6 +2079,7 @@ ostree_repo_set_collection_ref_immediate (OstreeRepo *self, * Note that if multiple threads are performing writes, all such threads must * have terminated before this function is invoked. * + * Locking: Releases `shared` lock acquired by `ostree_repo_prepare_transaction()` * Multithreading: This function is *not* MT safe; only one transaction can be * active at a time. */ diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 2ffd6948..4c883542 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -193,7 +193,7 @@ _ostree_repo_prune_tmp (OstreeRepo *self, * targeting that commit; otherwise any static delta of non existing commits are * deleted. * - * This function takes an exclusive lock on the @self repository. + * Locking: exclusive */ gboolean ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, @@ -326,7 +326,7 @@ repo_prune_internal (OstreeRepo *self, * statistics on objects that would be deleted, without actually * deleting them. * - * This function takes an exclusive lock on the @self repository. + * Locking: exclusive */ gboolean ostree_repo_prune (OstreeRepo *self, @@ -438,7 +438,7 @@ ostree_repo_prune (OstreeRepo *self, * The %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE flag may be specified to just determine * statistics on objects that would be deleted, without actually deleting them. * - * This function takes an exclusive lock on the @self repository. + * Locking: exclusive */ gboolean ostree_repo_prune_from_reachable (OstreeRepo *self, From d0a18323474e33d55da201395e3506f81021bce7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 2 May 2018 11:59:21 -0400 Subject: [PATCH 67/80] deploy: Use fdatasync() for new kernel/initramfs by default While we do a `syncfs()` plus `FIFREEZE/THAW` for `/boot`, that only comes during deployment finalization. The code here today generally assumes that if the file exists it's been fully written. So let's do a `fdatasync()` before we do the `rename()`. This just came out of looking through the code while working on deployment staging. In that scenario there's a much larger window between when we copy the kernel/initramfs and when we sync `/boot`. Closes: #1571 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index b21be0e9..7830e830 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -128,7 +128,7 @@ install_into_boot (OstreeSePolicy *sepolicy, error)) return FALSE; return glnx_file_copy_at (src_dfd, src_subpath, NULL, dest_dfd, dest_subpath, - GLNX_FILE_COPY_NOXATTRS, + GLNX_FILE_COPY_NOXATTRS | GLNX_FILE_COPY_DATASYNC, cancellable, error); } else From 490cfbe2262d6e3d4a4a873986fbc80c2b084629 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 May 2018 21:55:54 -0400 Subject: [PATCH 68/80] README: Tweak licensing description Basically it wasn't clear that the man pages and API doc remain LGPLv2. Closes: https://github.com/ostreedev/ostree/issues/1456 Closes: #1574 Approved by: smcv --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 485bc26d..c54151d1 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,10 @@ Licensing The licensing for the *code* of libostree can be canonically found in the individual files; and the overall status in the [COPYING](https://github.com/ostreedev/ostree/blob/master/COPYING) -file in the source. Currently, that's LGPLv2+. +file in the source. Currently, that's LGPLv2+. This also covers the man pages and API docs. -The license for the *documentation* of libostree is: `SPDX-License-Identifier: (CC-BY-SA-3.0 OR GFDL-1.3-or-later)` +The license for the manual documentation in the `doc/` directory is: +`SPDX-License-Identifier: (CC-BY-SA-3.0 OR GFDL-1.3-or-later)` +This is intended to allow use by Wikipedia and other projects. + +In general, files should have a `SPDX-License-Identifier` and that is canonical. From 956618a188b3da591694ed3172ebf1622294bd16 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 May 2018 21:32:56 -0400 Subject: [PATCH 69/80] tests: Tweak staged-deploy test to be faster Work around https://github.com/ostreedev/ostree/issues/1569 Closes: #1573 Approved by: jlebon --- tests/installed/destructive/staged-deploy.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index f946e139..3720d2d6 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -3,7 +3,10 @@ - name: Write staged-deploy commit shell: | - ostree --repo=/ostree/repo commit --parent="${commit}" -b staged-deploy --tree=ref="${commit}" --no-bindings + cd /ostree/repo/tmp + # https://github.com/ostreedev/ostree/issues/1569 + ostree checkout -H ${commit} t + ostree commit --no-bindings --parent="${commit}" -b staged-deploy -I --consume t newcommit=$(ostree rev-parse staged-deploy) orig_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) ostree admin deploy --stage staged-deploy @@ -51,7 +54,7 @@ shell: | ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment - ostree --repo=/ostree/repo commit --parent="${commit}" -b nonstaged-deploy --tree=ref="${commit}" --no-bindings + ostree --repo=/ostree/repo refs --create nonstaged-deploy "${commit}" ostree admin deploy nonstaged-deploy ostree admin status > status.txt grep -vqFe '(staged)' status.txt From bd904b26e1adcaa6f0d79482cfc5a9ab3c170594 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 4 May 2018 10:13:13 -0400 Subject: [PATCH 70/80] lib/deploy: Add semi-colon in post-deployment msg Let's add a semi-colon between the "bootconfig swap" part and the "deployment count change" to make it more clear they're separate statements. Closes: #1575 Approved by: cgwalters --- src/libostree/ostree-sysroot-deploy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 7830e830..6f0f6efd 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2352,7 +2352,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, } { g_autofree char *msg = - g_strdup_printf ("%s; bootconfig swap: %s deployment count change: %i", + g_strdup_printf ("%s; bootconfig swap: %s; deployment count change: %i", (bootloader_is_atomic ? "Transaction complete" : "Bootloader updated"), requires_new_bootversion ? "yes" : "no", new_deployments->len - self->deployments->len); From 589e97dc602f86f51a348da7d3c293520a3a80a0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 4 May 2018 10:13:14 -0400 Subject: [PATCH 71/80] lib/commit: Fix function name typo in docstring Closes: #1575 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 2ddcf498..86ee5e30 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1510,7 +1510,7 @@ devino_cache_lookup (OstreeRepo *self, * There is an upfront cost to creating this mapping, as this will scan the * entire objects directory. If your commit is composed of mostly hardlinks to * existing ostree objects, then this will speed up considerably, so call it - * before you call ostree_write_directory_to_mtree() or similar. However, + * before you call ostree_repo_write_directory_to_mtree() or similar. However, * ostree_repo_devino_cache_new() is better as it avoids scanning all objects. * * Multithreading: This function is *not* MT safe. From 579faf92fd668082e63520cb4ba324754d2b00bd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 4 May 2018 13:51:13 -0400 Subject: [PATCH 72/80] tests: Fix installed tests more OK so I noticed that something was failing and we were missing `set -xeuo pipefail` in our shells. That of course revealed the ansible tests didn't actually work - my only defense here is spending so much time fighting to get it through CI and trying something new. Anyways, to make the staged-deploy tests work we need a task that actually uses `rpm-ostree override` rather than `usroverlay`. Let's make this a bit saner and have a clean split between tests that are "shell-script+usroverlay" and "ansible+override". Closes: #1577 Approved by: jlebon --- tests/installed/destructive-ansible.yml | 17 +++++++++++++++++ .../{destructive.yml => destructive-unit.yml} | 8 ++++---- tests/installed/destructive/staged-deploy.yml | 10 +++++++--- tests/installed/run.sh | 2 +- tests/installed/tasks/install-git.yml | 19 +++++++++++++++++++ 5 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 tests/installed/destructive-ansible.yml rename tests/installed/{destructive.yml => destructive-unit.yml} (81%) create mode 100644 tests/installed/tasks/install-git.yml diff --git a/tests/installed/destructive-ansible.yml b/tests/installed/destructive-ansible.yml new file mode 100644 index 00000000..12ee5d52 --- /dev/null +++ b/tests/installed/destructive-ansible.yml @@ -0,0 +1,17 @@ +# Ansible-based tests. +--- +- hosts: localhost + tags: + - atomic + remote_user: root + vars: + use_git_build: True + tests: "." + tasks: + - import_tasks: tasks/query-host.yml + - set_fact: + rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}" + - import_tasks: tasks/install-git.yml + when: use_git_build + + - import_tasks: destructive/staged-deploy.yml diff --git a/tests/installed/destructive.yml b/tests/installed/destructive-unit.yml similarity index 81% rename from tests/installed/destructive.yml rename to tests/installed/destructive-unit.yml index 5bd4d7a7..e70561d4 100644 --- a/tests/installed/destructive.yml +++ b/tests/installed/destructive-unit.yml @@ -1,4 +1,7 @@ -# This entrypoint right now just runs the sysinstalled-tests. +# This entrypoint right now just runs shell-script based tests +# from destructive/. Note that we `rpm-ostree usroverlay` git +# builds. So it's not supported to reboot in these tests. +# These tests will be run serially, and can e.g. change deployments. --- - hosts: localhost tags: @@ -17,9 +20,6 @@ - name: Copy test data synchronize: src=../../ dest=/root/tests/ archive=yes - # First, the Ansible-based tests - - import_tasks: destructive/staged-deploy.yml - - find: paths: /root/tests/installed/destructive patterns: "itest-*.sh" diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index 3720d2d6..723dbf55 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -3,6 +3,7 @@ - name: Write staged-deploy commit shell: | + set -xeuo pipefail cd /ostree/repo/tmp # https://github.com/ostreedev/ostree/issues/1569 ostree checkout -H ${commit} t @@ -11,7 +12,7 @@ orig_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) ostree admin deploy --stage staged-deploy new_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) - assert_not_streq "${orig_mtime}" "${new_mtime}" + test "${orig_mtime}" != "${new_mtime}" test -f /run/ostree/staged-deployment ostree refs | grep -E -e '^ostree/' | while read ref; do if test "$(ostree rev-parse ${ref})" = "${newcommit}"; then @@ -25,6 +26,7 @@ - include_tasks: ../tasks/reboot.yml - name: Check that deploy-staged service worked shell: | + set -xeuo pipefail # Assert that the previous boot had a journal entry for it journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' # And there should not be a staged deployment @@ -33,11 +35,13 @@ shell: rpm-ostree rollback - include_tasks: ../tasks/reboot.yml - shell: | + set -xeuo pipefail rpm-ostree cleanup -rp # Ensure we can unstage - name: Write staged-deploy commit, then unstage shell: | + set -xeuo pipefail ostree admin deploy --stage staged-deploy ostree admin status > status.txt grep -qFe '(staged)' status.txt @@ -49,9 +53,9 @@ environment: commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" -# Staged should be overwritten by non-staged -- name: Write staged-deploy commit, then unstage +- name: Staged should be overwritten by non-staged shell: | + set -xeuo pipefail ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment ostree --repo=/ostree/repo refs --create nonstaged-deploy "${commit}" diff --git a/tests/installed/run.sh b/tests/installed/run.sh index 264a7436..2ee4a3dd 100755 --- a/tests/installed/run.sh +++ b/tests/installed/run.sh @@ -6,7 +6,7 @@ set -xeuo pipefail dn=$(cd $(dirname $0) && pwd) # TODO: parallelize this -PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive.yml} +PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive-ansible.yml destructive-unit.yml} for playbook in $PLAYBOOKS; do time ${dn}/playbook-run.sh -v ${dn}/${playbook} done diff --git a/tests/installed/tasks/install-git.yml b/tests/installed/tasks/install-git.yml new file mode 100644 index 00000000..33fe385e --- /dev/null +++ b/tests/installed/tasks/install-git.yml @@ -0,0 +1,19 @@ +# Use package layering to install git builds. +- command: ostree --version + changed_when: False + register: ostree_orig_version +- set_fact: + ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}" +- name: Copy locally built RPMs + synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes +- name: Install RPMs + shell: rpm-ostree override replace /root/x86_64/*.rpm +- include_tasks: ../tasks/reboot.yml +- command: ostree --version + register: ostree_new_version +- set_fact: + ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}" +- name: "Fail if we didn't change the ostree version" + when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] + fail: + msg: "Failed to change ostree version" From 5ee78ea42a6dd2cb63e56652298350512fd56c7b Mon Sep 17 00:00:00 2001 From: Tobias Mueller Date: Sat, 5 May 2018 22:00:06 +0200 Subject: [PATCH 73/80] repo: handle GPG_ERR_AMBIGUOUS_NAME in sign_data This should give a more insightful error message if the user provides a UID which is present on multiple keys. This happens if you have an old key in your keyring which you are not actively using any more, e.g. because it is too old. You still have your old keys in your keyring, because you want to read old email encrypted for that key, though. The gpgme function used by ostree right now complains if a UID is found on multiple keys: https://www.gnupg.org/documentation/manuals/gpgme/Listing-Keys.html#index-gpgme_005fget_005fkey The used API is too simple for that use case. Note that it would be nicer if ostree picked the only valid signing key out of the available keys rather than using the simplistic gpgme_get_key function. It be nicer, of course, if there was such a gpgme function. Closes: #1579 Approved by: cgwalters --- src/libostree/ostree-repo.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 972f05ae..3251880f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4640,6 +4640,10 @@ sign_data (OstreeRepo *self, if (gpgme_err_code (err) == GPG_ERR_EOF) return glnx_throw (error, "No gpg key found with ID %s (homedir: %s)", key_id, homedir ? homedir : ""); + else if (gpgme_err_code (err) == GPG_ERR_AMBIGUOUS_NAME) { + return glnx_throw (error, "gpg key id %s ambiguous (homedir: %s). Try the fingerprint instead", key_id, + homedir ? homedir : ""); + } else if (err != GPG_ERR_NO_ERROR) return ot_gpgme_throw (err, error, "Unable to lookup key ID %s", key_id); From a3295ee584a107784c34fea8616140b1e6ecdf94 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 7 May 2018 21:02:12 -0700 Subject: [PATCH 74/80] ci: Drop str hotpatch The change we want is in the current Fedora repos, and git master is broken: > qemu-system-x86_64: -vnc :1: Failed to start VNC server: Failed to bind socket: Address already in use https://pagure.io/standard-test-roles/pull-request/186#comment-52440 Closes: #1582 Approved by: jlebon --- tests/installed/provision.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/installed/provision.sh b/tests/installed/provision.sh index 3a1efe94..b8ecdb48 100755 --- a/tests/installed/provision.sh +++ b/tests/installed/provision.sh @@ -6,7 +6,3 @@ dn=$(dirname $0) pkg_upgrade pkg_install git rsync openssh-clients ansible standard-test-roles - -# "Hot patch" this to pick up https://pagure.io/standard-test-roles/pull-request/152 -# so we get parallelism -cd /usr/share/ansible/inventory && curl -L -O https://pagure.io/standard-test-roles/raw/master/f/inventory/standard-inventory-qcow2 From 053efeb23a16c63028e25d42eb942d8f84ef5534 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 7 May 2018 09:41:27 -0400 Subject: [PATCH 75/80] docs: Add "Hello World" example Let's get practical faster in the manual and have a simple "Hello World" example right off the bat to hopefully make it easier to grok how OSTree works. Also some minor tweaks on wording around comparisons to git. Closes: #1581 Approved by: cgwalters --- README.md | 6 ++-- docs/manual/introduction.md | 71 ++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c54151d1..36bcfc24 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ bootloader configuration. The core OSTree model is like git in that it checksums individual files and has a content-addressed-object store. It's unlike git in that it "checks out" the -files via hardlinks, and they should thus be immutable. Therefore, another way -to think of OSTree is that it's just a more polished version -of +files via hardlinks, and they thus need to be immutable to prevent corruption. +Therefore, another way to think of OSTree is that it's just a more polished +version of [Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285). **Features:** diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md index 92fc45f2..c0113f5d 100644 --- a/docs/manual/introduction.md +++ b/docs/manual/introduction.md @@ -11,15 +11,78 @@ replicating them to clients. The underlying architecture might be summarized as "git for operating system binaries". It operates in userspace, and will work on top of any Linux filesystem. At its core is a git-like -content-addressed object store, and layered on top of that is -bootloader configuration, management of -`/etc`, and other functions to perform an -upgrade beyond just replicating files. +content-addressed object store with branches (or "refs") to track +meaningful filesystem trees within the store. Similarly, one can +check out or commit to these branches. + +Layered on top of that is bootloader configuration, management of +`/etc`, and other functions to perform an upgrade beyond just +replicating files. You can use OSTree standalone in the pure replication model, but another approach is to add a package manager on top, thus creating a hybrid tree/package system. +## Hello World example + +OSTree is mostly used as a library, but a quick tour of using its +CLI tools can give a general idea of how it works at its most +basic level. + +You can create a new OSTree repository using `init`: + +``` +$ ostree --repo=repo init +``` + +This will create a new `repo` directory containing your +repository. Feel free to inspect it. + +Now, let's prepare some data to add to the repo: + +``` +$ mkdir tree +$ echo "Hello world!" > tree/hello.txt +``` + +We can now import our `tree/` directory using the `commit` +command: + +``` +$ ostree --repo=repo commit --branch=foo tree/ +``` + +This will create a new branch `foo` pointing to the full tree +imported from `tree/`. In fact, we could now delete `tree/` if we +wanted to. + +To check that we indeed now have a `foo` branch, you can use the +`refs` command: + +``` +$ ostree --repo=repo refs +foo +``` + +We can also inspect the filesystem tree using the `ls` and `cat` +commands: + +``` +$ ostree --repo=repo ls foo +d00775 1000 1000 0 / +-00664 1000 1000 13 /hello.txt +$ ostree --repo=repo cat foo /hello.txt +Hello world! +``` + +And finally, we can check out our tree from the repository: + +``` +$ ostree --repo=repo checkout foo tree-checkout/ +$ cat tree-checkout/hello.txt +Hello world! +``` + ## Comparison with "package managers" Because OSTree is designed for deploying core operating From 44706430f4fa4a7a6ab520aa450d8111b5f66b1b Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 7 May 2018 08:55:15 -0400 Subject: [PATCH 76/80] libglnx: Bump to fix F28 compilation Pull in https://github.com/GNOME/libglnx/pull/104 to fix compiling on F28. Update submodule: libglnx Closes: #1580 Approved by: cgwalters --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 0c82203c..03e16afa 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 0c82203cd459a35cc3f471e3205355e9fb79160f +Subproject commit 03e16afa7f2cf7ce517b604035683fa0409f730c From ba628da9ad1052d233b20527da9c3ae3c4775bd9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 9 May 2018 14:45:32 -0700 Subject: [PATCH 77/80] ci: Switch libgcrypt URL Since the previous one wasn't made into an update, it got GC'd. Closes: #1584 Approved by: jlebon --- ci/libpaprci/libbuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh index 6ce7dcec..8b8af3f5 100644 --- a/ci/libpaprci/libbuild.sh +++ b/ci/libpaprci/libbuild.sh @@ -57,7 +57,7 @@ pkg_builddep() { fi # https://bugzilla.redhat.com/show_bug.cgi?id=1542453 if rpm -q libgcrypt | grep -q libgcrypt-1.8.2-1.fc27; then - dnf install -y https://kojipkgs.fedoraproject.org//packages/libgcrypt/1.8.2/2.fc27/x86_64/libgcrypt-1.8.2-2.fc27.x86_64.rpm + dnf install -y https://fedorapeople.org/~walters/libgcrypt-1.8.2-2.fc27.x86_64.rpm fi else yum-builddep -y "$@" From 605d3132062e1e0b35a21578adb04a5eaae8c6d4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 9 May 2018 07:46:37 -0700 Subject: [PATCH 78/80] tests/installed: Disable all rpmmd repos For the same reason we do in the rpm-ostree tests. This also made sure the test run worked when I was offline on a plane. Closes: #1583 Approved by: jlebon --- tests/installed/destructive-ansible.yml | 1 + tests/installed/tasks/disable-all-rpmmd-repos.yml | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/installed/tasks/disable-all-rpmmd-repos.yml diff --git a/tests/installed/destructive-ansible.yml b/tests/installed/destructive-ansible.yml index 12ee5d52..a0e69aa1 100644 --- a/tests/installed/destructive-ansible.yml +++ b/tests/installed/destructive-ansible.yml @@ -8,6 +8,7 @@ use_git_build: True tests: "." tasks: + - import_tasks: tasks/disable-all-rpmmd-repos.yml - import_tasks: tasks/query-host.yml - set_fact: rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}" diff --git a/tests/installed/tasks/disable-all-rpmmd-repos.yml b/tests/installed/tasks/disable-all-rpmmd-repos.yml new file mode 100644 index 00000000..b4f2a074 --- /dev/null +++ b/tests/installed/tasks/disable-all-rpmmd-repos.yml @@ -0,0 +1,5 @@ +- name: Disable all rpmmd repos + shell: | + for x in /etc/yum.repos.d/*.repo; do + sed -i -e 's,^enabled=,enabled=0,g' $x + done From 6873650cae60b1c5ff57366d133b74b40803417b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 May 2018 17:29:06 -0400 Subject: [PATCH 79/80] lib/sysroot: Add OSTREE_EX_STAGE_DEPLOYMENTS environment variable I feel like I'm drowning in a pile of experimental-but-almost-stable features... Anyways, since we made the feature opt-in in rpm-ostree in https://github.com/projectatomic/rpm-ostree/pull/1352 let's mirror that a bit here with an environment variable so people can play with it more easily. The tests needed some tweaks; specifically we need to reload the status fact after making changes. I'm still a bit uncertain about the Ansible-as-tests. But we add an upgrade test that uses the new environment variable. Closes: #1583 Approved by: jlebon --- src/libostree/ostree-sysroot-upgrader.c | 44 +++++++++++++------ tests/admin-test.sh | 7 ++- tests/installed/destructive-ansible.yml | 5 +-- tests/installed/destructive/staged-deploy.yml | 25 ++++++++--- tests/installed/tasks/install-git.yml | 3 +- tests/installed/tasks/query-host.yml | 3 +- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index a87548a3..8fb231a3 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -644,21 +644,37 @@ ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader *self, GError **error) { g_autoptr(OstreeDeployment) new_deployment = NULL; - if (!ostree_sysroot_deploy_tree (self->sysroot, self->osname, - self->new_revision, - self->origin, - self->merge_deployment, - NULL, - &new_deployment, - cancellable, error)) - return FALSE; - if (!ostree_sysroot_simple_write_deployment (self->sysroot, self->osname, - new_deployment, - self->merge_deployment, - 0, - cancellable, error)) - return FALSE; + /* Experimental flag to enable staging */ + if (getenv ("OSTREE_EX_STAGE_DEPLOYMENTS")) + { + if (!ostree_sysroot_stage_tree (self->sysroot, self->osname, + self->new_revision, + self->origin, + self->merge_deployment, + NULL, + &new_deployment, + cancellable, error)) + return FALSE; + } + else + { + if (!ostree_sysroot_deploy_tree (self->sysroot, self->osname, + self->new_revision, + self->origin, + self->merge_deployment, + NULL, + &new_deployment, + cancellable, error)) + return FALSE; + + if (!ostree_sysroot_simple_write_deployment (self->sysroot, self->osname, + new_deployment, + self->merge_deployment, + 0, + cancellable, error)) + return FALSE; + } return TRUE; } diff --git a/tests/admin-test.sh b/tests/admin-test.sh index dd87cd11..7384d8f3 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((24 + ${extra_admin_tests:-0}))" +echo "1..$((25 + ${extra_admin_tests:-0}))" function validate_bootloader() { cd ${test_tmpdir}; @@ -215,6 +215,11 @@ validate_bootloader echo "ok upgrade bare" os_repository_new_commit +if env OSTREE_EX_STAGE_DEPLOYMENTS=1 ${CMD_PREFIX} ostree admin upgrade --os=testos 2>err.txt; then + fatal "staged when not booted" +fi +echo "ok upgrade failed when staged" + ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo remote add --set=gpg-verify=false testos file://$(pwd)/testos-repo testos/buildmaster/x86_64-runtime ${CMD_PREFIX} ostree admin upgrade --os=testos origrev=${rev} diff --git a/tests/installed/destructive-ansible.yml b/tests/installed/destructive-ansible.yml index a0e69aa1..c72f6b82 100644 --- a/tests/installed/destructive-ansible.yml +++ b/tests/installed/destructive-ansible.yml @@ -9,10 +9,7 @@ tests: "." tasks: - import_tasks: tasks/disable-all-rpmmd-repos.yml - - import_tasks: tasks/query-host.yml - - set_fact: - rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}" - import_tasks: tasks/install-git.yml when: use_git_build - + - import_tasks: tasks/query-host.yml - import_tasks: destructive/staged-deploy.yml diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/installed/destructive/staged-deploy.yml index 723dbf55..f34550a2 100644 --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/installed/destructive/staged-deploy.yml @@ -27,16 +27,31 @@ - name: Check that deploy-staged service worked shell: | set -xeuo pipefail + rpm-ostree status # Assert that the previous boot had a journal entry for it journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' # And there should not be a staged deployment test '!' -f /run/ostree/staged-deployment -- name: Rollback - shell: rpm-ostree rollback -- include_tasks: ../tasks/reboot.yml -- shell: | + +- name: Upgrade with staging + shell: | set -xeuo pipefail + test '!' -f /run/ostree/staged-deployment + ostree admin deploy --stage staged-deploy + test -f /run/ostree/staged-deployment + origcommit=$(ostree rev-parse staged-deploy) + cd /ostree/repo/tmp + ostree checkout -H "${origcommit}" t + ostree commit --no-bindings --parent="${origcommit}" -b staged-deploy -I --consume t + newcommit=$(ostree rev-parse staged-deploy) + env OSTREE_EX_STAGE_DEPLOYMENTS=1 ostree admin upgrade >out.txt + test -f /run/ostree/staged-deployment + # Debating bouncing back out to Ansible for this + firstdeploycommit=$(rpm-ostree status |grep 'Commit:' |head -1|sed -e 's,^ *Commit: *,,') + test "${firstdeploycommit}" = "${newcommit}" + # Cleanup rpm-ostree cleanup -rp +- import_tasks: ../tasks/query-host.yml # Ensure we can unstage - name: Write staged-deploy commit, then unstage @@ -50,8 +65,6 @@ ostree admin status > status.txt grep -vqFe '(staged)' status.txt test '!' -f /run/ostree/staged-deployment - environment: - commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" - name: Staged should be overwritten by non-staged shell: | diff --git a/tests/installed/tasks/install-git.yml b/tests/installed/tasks/install-git.yml index 33fe385e..8216afeb 100644 --- a/tests/installed/tasks/install-git.yml +++ b/tests/installed/tasks/install-git.yml @@ -8,7 +8,8 @@ synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes - name: Install RPMs shell: rpm-ostree override replace /root/x86_64/*.rpm -- include_tasks: ../tasks/reboot.yml +- import_tasks: ../tasks/reboot.yml +- import_tasks: ../tasks/query-host.yml - command: ostree --version register: ostree_new_version - set_fact: diff --git a/tests/installed/tasks/query-host.yml b/tests/installed/tasks/query-host.yml index 2a67dbdf..d572ae64 100644 --- a/tests/installed/tasks/query-host.yml +++ b/tests/installed/tasks/query-host.yml @@ -1,4 +1,5 @@ -- command: rpm-ostree status --json +- name: Load status json + command: rpm-ostree status --json changed_when: False register: rpmostree_status_json - set_fact: From 986f176e7cd40618f40579a33b4a82cafeeee1df Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 11 May 2018 08:59:31 -0400 Subject: [PATCH 80/80] Release 2018.5 Lots of stuff queued up. Closes: #1585 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 8 -------- src/libostree/libostree-released.sym | 9 +++++++++ tests/test-symbols.sh | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index 6c7a8b5c..afedc030 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ m4_define([year_version], [2018]) m4_define([release_version], [5]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 226b2ead..46ad280c 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,14 +18,6 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2018.5 { - ostree_sysroot_stage_tree; - ostree_sysroot_get_staged_deployment; - ostree_deployment_is_staged; - ostree_repo_traverse_new_parents; - ostree_repo_traverse_parents_get_commits; - ostree_repo_traverse_commit_union_with_parents; -} LIBOSTREE_2018.3; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index e9a95cc7..533bc5c2 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -464,6 +464,15 @@ LIBOSTREE_2018.3 { ostree_deployment_is_pinned; } LIBOSTREE_2018.2; +LIBOSTREE_2018.5 { + ostree_sysroot_stage_tree; + ostree_sysroot_get_staged_deployment; + ostree_deployment_is_staged; + ostree_repo_traverse_new_parents; + ostree_repo_traverse_parents_get_commits; + ostree_repo_traverse_commit_union_with_parents; +} LIBOSTREE_2018.3; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 13b47afb..501bedeb 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -54,7 +54,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <