From 4d6e8f2b995e0b10a4b8a4f8f75324cd4bd0aaa0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 Sep 2020 18:00:27 +0000 Subject: [PATCH 01/58] Post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 0c3c6fed..c81507d2 100644 --- a/configure.ac +++ b/configure.ac @@ -7,10 +7,10 @@ dnl Seed the release notes with `git-shortlog-with-prs ..`. Th dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2020]) -m4_define([release_version], [6]) +m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From be0f9e77e5d99e1315688d282f7cce57ff7fd484 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 3 Sep 2020 22:14:02 +0000 Subject: [PATCH 02/58] ci: Drop var mount test Merged in https://github.com/coreos/fedora-coreos-config/pull/586 --- .cci.jenkinsfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 5c6bb21d..ac65b9c8 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -72,12 +72,6 @@ parallel fcos: { tar -C insttree -xzvf insttree.tar.gz rsync -rlv insttree/ / coreos-assembler init --force https://github.com/coreos/fedora-coreos-config - # XXX: We temporarily add these tests until they get merged into FCOS proper - (mkdir -p tests/kola/var-mount - cd tests/kola/var-mount - curl -L --remote-name-all \ - https://raw.githubusercontent.com/jlebon/fedora-coreos-config/pr/var-mount/tests/kola/var-mount/{config.ign,test.sh} - chmod a+x test.sh) mkdir -p overrides/rootfs mv insttree/* overrides/rootfs/ rmdir insttree From a1bd29f2454f3a2b86d66226e918057e8eccc2cd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 9 Sep 2020 12:34:44 +0000 Subject: [PATCH 03/58] deploy: Add some error prefixing around xattr setting Looking at https://github.com/coreos/coreos-assembler/issues/1703 a user is getting a bare: `error: fsetxattr: Permission denied` I don't think it's these code paths since a deploy isn't happening but on inspection I noticed we didn't have error prefixing here. --- src/libostree/ostree-sysroot-deploy.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 0e02d1f2..65d54b46 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -313,7 +313,7 @@ copy_dir_recurse (int src_parent_dfd, if (!dirfd_copy_attributes_and_xattrs (src_parent_dfd, name, src_dfd_iter.fd, dest_dfd, flags, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Copying attributes of %s", name); while (TRUE) { @@ -340,7 +340,7 @@ copy_dir_recurse (int src_parent_dfd, dest_dfd, dent->d_name, sysroot_flags_to_copy_flags (GLNX_FILE_COPY_OVERWRITE, flags), cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Copying %s", dent->d_name); } } @@ -488,7 +488,7 @@ copy_modified_config_file (int orig_etc_fd, new_etc_fd, path, sysroot_flags_to_copy_flags (GLNX_FILE_COPY_OVERWRITE, flags), cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Copying %s", path); } else { From 74bae256feaf297b80f0898c48ab762983b6c69a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 31 Aug 2020 17:00:39 +0200 Subject: [PATCH 04/58] list-deltas: Don't break on non-subdir entries ostree_repo_list_static_delta_names() tried to validate that any second-level directory element was a directory, but there was a cut-and-paste issue, and it used `dent->d_type` instead of `sub_dent->d_type`. This fixes the code, but all old ostree versions will break if there are non-directories in a subdirectory of the deltas directory in the repo, so be wary. --- src/libostree/ostree-repo-static-delta-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index ade4e9df..d12bf439 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -109,7 +109,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, return FALSE; if (sub_dent == NULL) break; - if (dent->d_type != DT_DIR) + if (sub_dent->d_type != DT_DIR) continue; const char *name1 = dent->d_name; From bb2649a8c087105542edccbead26b6d9b8fc366a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 8 Sep 2020 11:37:33 +0200 Subject: [PATCH 05/58] Fix leak when signing _ostree_detached_metadata_append_gpg_sig() was returning a floating ref, but all users were using g_autoptr. Fix it by adding a ref-sink. --- src/libostree/ostree-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 523f57c0..29528fa5 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -2675,7 +2675,7 @@ _ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata, _OSTREE_METADATA_GPGSIGS_NAME, g_variant_builder_end (signature_builder)); - return g_variant_dict_end (&metadata_dict); + return g_variant_ref_sink (g_variant_dict_end (&metadata_dict)); } #endif /* OSTREE_DISABLE_GPGME */ From 85accb84e87a4964a0fff02b28117a1e6b137c39 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 11 Sep 2020 11:22:49 +0200 Subject: [PATCH 06/58] pull: Break out _ostree_repo_save_cache_summary_file() helper This is a minor cleanup as its just called twice from _ostree_repo_cache_summary(). However, later code will need it in more places. --- src/libostree/ostree-repo-pull.c | 52 ++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index d817575b..0e342568 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2677,6 +2677,34 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, return TRUE; } +static gboolean +_ostree_repo_save_cache_summary_file (OstreeRepo *self, + const char *filename, + const char *extension, + GBytes *data, + GCancellable *cancellable, + GError **error) +{ + const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension); + glnx_autofd int fd = -1; + + if (self->cache_dir_fd == -1) + return TRUE; + + if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, DEFAULT_DIRECTORY_MODE, cancellable, error)) + return FALSE; + + if (!glnx_file_replace_contents_at (self->cache_dir_fd, + file, + g_bytes_get_data (data, NULL), + g_bytes_get_size (data), + self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + return FALSE; + + return TRUE; +} + /* Replace the current summary+signature with new versions */ static gboolean _ostree_repo_cache_summary (OstreeRepo *self, @@ -2686,28 +2714,12 @@ _ostree_repo_cache_summary (OstreeRepo *self, GCancellable *cancellable, GError **error) { - if (self->cache_dir_fd == -1) - return TRUE; - - if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, DEFAULT_DIRECTORY_MODE, cancellable, error)) + if (!_ostree_repo_save_cache_summary_file (self, remote, NULL, + summary, cancellable, error)) return FALSE; - const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote); - if (!glnx_file_replace_contents_at (self->cache_dir_fd, - summary_cache_file, - g_bytes_get_data (summary, NULL), - g_bytes_get_size (summary), - self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, - cancellable, error)) - return FALSE; - - const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig"); - if (!glnx_file_replace_contents_at (self->cache_dir_fd, - summary_cache_sig_file, - g_bytes_get_data (summary_sig, NULL), - g_bytes_get_size (summary_sig), - self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, - cancellable, error)) + if (!_ostree_repo_save_cache_summary_file (self, remote, ".sig", + summary_sig, cancellable, error)) return FALSE; return TRUE; From dddb449d2c099ffff5bb4f9de2ed4d2c61b4e905 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 11 Sep 2020 11:24:43 +0200 Subject: [PATCH 07/58] pull: Actually mmap summary files The change in cbf1aca1d5c08d2f40832d16670484ba878d95fb actually only mmaps the signature file, not the summary. This change makes use mmap both, as well as extract the cache loading into a helper function that we will later use in more places. --- src/libostree/ostree-repo-pull.c | 86 ++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0e342568..f7f34d64 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2626,54 +2626,78 @@ validate_variant_is_csum (GVariant *csum, return ostree_validate_structureof_csum_v (csum, error); } +static gboolean +_ostree_repo_load_cache_summary_file (OstreeRepo *self, + const char *filename, + const char *extension, + GBytes **out_data, + GCancellable *cancellable, + GError **error) +{ + const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension); + glnx_autofd int fd = -1; + g_autoptr(GBytes) data = NULL; + + *out_data = NULL; + + if (self->cache_dir_fd == -1) + return TRUE; + + fd = openat (self->cache_dir_fd, file, O_CLOEXEC | O_RDONLY); + if (fd < 0) + { + if (errno == ENOENT) + return TRUE; + return glnx_throw_errno_prefix (error, "openat(%s)", file); + } + + data = ot_fd_readall_or_mmap (fd, 0, error); + if (!data) + return FALSE; + + *out_data =g_steal_pointer (&data); + return TRUE; +} + /* Load the summary from the cache if the provided .sig file is the same as the cached version. */ static gboolean _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, const char *remote, GBytes *summary_sig, - GBytes **summary, + GBytes **out_summary, GCancellable *cancellable, GError **error) { - if (self->cache_dir_fd == -1) - return TRUE; + g_autoptr(GBytes) old_sig_contents = NULL; - const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig"); - glnx_autofd int prev_fd = -1; - if (!ot_openat_ignore_enoent (self->cache_dir_fd, summary_cache_sig_file, &prev_fd, error)) - return FALSE; - if (prev_fd < 0) - return TRUE; /* Note early return */ + *out_summary = NULL; - g_autoptr(GBytes) old_sig_contents = ot_fd_readall_or_mmap (prev_fd, 0, error); - if (!old_sig_contents) + if (!_ostree_repo_load_cache_summary_file (self, remote, ".sig", + &old_sig_contents, + cancellable, error)) return FALSE; - if (g_bytes_compare (old_sig_contents, summary_sig) == 0) + if (old_sig_contents != NULL && + g_bytes_compare (old_sig_contents, summary_sig) == 0) { - const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote); - glnx_autofd int summary_fd = -1; - GBytes *summary_data; + g_autoptr(GBytes) summary_data = NULL; - - summary_fd = openat (self->cache_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY); - if (summary_fd < 0) - { - if (errno == ENOENT) - { - (void) unlinkat (self->cache_dir_fd, summary_cache_sig_file, 0); - return TRUE; /* Note early return */ - } - - return glnx_throw_errno_prefix (error, "openat(%s)", summary_cache_file); - } - - summary_data = glnx_fd_readall_bytes (summary_fd, cancellable, error); - if (!summary_data) + if (!_ostree_repo_load_cache_summary_file (self, remote, NULL, + &summary_data, + cancellable, error)) return FALSE; - *summary = summary_data; + + if (summary_data == NULL) + { + /* Cached signature without cached summary, remove the signature */ + const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig"); + (void) unlinkat (self->cache_dir_fd, summary_cache_sig_file, 0); + } + else + *out_summary = g_steal_pointer (&summary_data); } + return TRUE; } From 32014d99e617a1a85f6ff4f099ac406e6cc86eaa Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 14:29:47 +0200 Subject: [PATCH 08/58] Add and use ot_checksum_bytes helper This removes some duplicated code (and will be use even more later). --- src/libostree/ostree-repo-pull.c | 5 +---- src/libotutil/ot-checksum-utils.c | 10 ++++++++++ src/libotutil/ot-checksum-utils.h | 3 +++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f7f34d64..e54da67e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2513,10 +2513,7 @@ on_superblock_fetched (GObject *src, const guchar *expected_summary_digest = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta); guint8 actual_summary_digest[OSTREE_SHA256_DIGEST_LEN]; - g_auto(OtChecksum) hasher = { 0, }; - ot_checksum_init (&hasher); - ot_checksum_update_bytes (&hasher, delta_superblock_data); - ot_checksum_get_digest (&hasher, actual_summary_digest, sizeof (actual_summary_digest)); + ot_checksum_bytes (delta_superblock_data, actual_summary_digest); #ifndef OSTREE_DISABLE_GPGME /* At this point we've GPG verified the data, so in theory diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c index 66767368..26e0280a 100644 --- a/src/libotutil/ot-checksum-utils.c +++ b/src/libotutil/ot-checksum-utils.c @@ -275,3 +275,13 @@ ot_checksum_file_at (int dfd, ot_checksum_get_hexdigest (&checksum, hexdigest, sizeof (hexdigest)); return g_strdup (hexdigest); } + +void +ot_checksum_bytes (GBytes *data, + guint8 out_digest[_OSTREE_SHA256_DIGEST_LEN]) +{ + g_auto(OtChecksum) hasher = { 0, }; + ot_checksum_init (&hasher); + ot_checksum_update_bytes (&hasher, data); + ot_checksum_get_digest (&hasher, out_digest, _OSTREE_SHA256_DIGEST_LEN); +} diff --git a/src/libotutil/ot-checksum-utils.h b/src/libotutil/ot-checksum-utils.h index 411ef25d..5432c81e 100644 --- a/src/libotutil/ot-checksum-utils.h +++ b/src/libotutil/ot-checksum-utils.h @@ -96,4 +96,7 @@ char * ot_checksum_file_at (int dfd, GCancellable *cancellable, GError **error); +void ot_checksum_bytes (GBytes *data, + guint8 out_digest[_OSTREE_SHA256_DIGEST_LEN]); + G_END_DECLS From 598adc457fd624175822550e3878d0b605dce447 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 31 Aug 2020 17:21:43 +0200 Subject: [PATCH 09/58] deltas: Break out _ostree_repo_static_delta_superblock_digest() helper This loads and makes a digest for a delta superblock. The previous code was used when generating the deltas section in the summary file. This changes nothing, but is in preparation for using similar formats in a separate delta index file. --- src/libostree/ostree-repo-static-delta-core.c | 22 +++++++++++++++++++ .../ostree-repo-static-delta-private.h | 6 +++++ src/libostree/ostree-repo.c | 21 ++++++------------ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index d12bf439..835ec7f3 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -54,6 +54,28 @@ _ostree_static_delta_parse_checksum_array (GVariant *array, return TRUE; } +GVariant * +_ostree_repo_static_delta_superblock_digest (OstreeRepo *repo, + const char *from, + const char *to, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); + glnx_autofd int superblock_file_fd = -1; + guint8 digest[OSTREE_SHA256_DIGEST_LEN]; + + if (!glnx_openat_rdonly (repo->repo_dir_fd, superblock, TRUE, &superblock_file_fd, error)) + return NULL; + + g_autoptr(GBytes) superblock_content = ot_fd_readall_or_mmap (superblock_file_fd, 0, error); + if (!superblock_content) + return NULL; + + ot_checksum_bytes (superblock_content, digest); + + return ot_gvariant_new_bytearray (digest, sizeof (digest)); +} /** * ostree_repo_list_static_delta_names: diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 155acd52..ff8de9d4 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -190,6 +190,12 @@ _ostree_repo_static_delta_query_exists (OstreeRepo *repo, gboolean *out_exists, GCancellable *cancellable, GError **error); +GVariant * +_ostree_repo_static_delta_superblock_digest (OstreeRepo *repo, + const char *from, + const char *to, + GCancellable *cancellable, + GError **error); gboolean _ostree_repo_static_delta_dump (OstreeRepo *repo, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 95eb0efc..621668d7 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5793,25 +5793,18 @@ ostree_repo_regenerate_summary (OstreeRepo *self, { g_autofree char *from = NULL; g_autofree char *to = NULL; + GVariant *digest; + if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) return FALSE; - g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); - glnx_autofd int superblock_file_fd = -1; - - if (!glnx_openat_rdonly (self->repo_dir_fd, superblock, TRUE, &superblock_file_fd, error)) + digest = _ostree_repo_static_delta_superblock_digest (self, + (from && from[0]) ? from : NULL, + to, cancellable, error); + if (digest == NULL) return FALSE; - g_autoptr(GBytes) superblock_content = ot_fd_readall_or_mmap (superblock_file_fd, 0, error); - if (!superblock_content) - return FALSE; - g_auto(OtChecksum) hasher = { 0, }; - ot_checksum_init (&hasher); - ot_checksum_update_bytes (&hasher, superblock_content); - guint8 digest[OSTREE_SHA256_DIGEST_LEN]; - ot_checksum_get_digest (&hasher, digest, sizeof (digest)); - - g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], ot_gvariant_new_bytearray (digest, sizeof (digest))); + g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); } if (delta_names->len > 0) From 1f1ef4c98978faabc53c26878bd0af642d9b58b9 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 10:59:44 +0200 Subject: [PATCH 10/58] Break out the signature verification code into a helper function This changes nothing in the behaviour, but we want to later re-use this when we also verify the summary index. --- src/libostree/ostree-repo-pull.c | 139 ++++++++++++++++++------------- 1 file changed, 79 insertions(+), 60 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index e54da67e..ec9983fb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2623,6 +2623,80 @@ validate_variant_is_csum (GVariant *csum, return ostree_validate_structureof_csum_v (csum, error); } +static gboolean +_ostree_repo_verify_summary (OstreeRepo *self, + const char *name, + gboolean gpg_verify_summary, + GPtrArray *signapi_summary_verifiers, + GBytes *summary, + GBytes *signatures, + GCancellable *cancellable, + GError **error) +{ + if (gpg_verify_summary) + { + if (summary == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)"); + return FALSE; + } + + if (signatures == NULL) + { + g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, + "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); + return FALSE; + } + + /* Verify any summary signatures. */ + if (summary != NULL && signatures != NULL) + { + g_autoptr(OstreeGpgVerifyResult) result = NULL; + + result = ostree_repo_verify_summary (self, + name, + summary, + signatures, + cancellable, + error); + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + return FALSE; + } + } + + if (signapi_summary_verifiers) + { + if (summary == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Signature verification enabled, but no summary found (check that the configured URL in remote config is correct)"); + return FALSE; + } + + if (signatures == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Signature verification enabled, but no summary signatures found (use sign-verify-summary=false in remote config to disable)"); + return FALSE; + } + + /* Verify any summary signatures. */ + if (summary != NULL && signatures != NULL) + { + g_autoptr(GVariant) sig_variant = NULL; + + sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + signatures, FALSE); + + if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, NULL, error)) + return FALSE; + } + } + + return TRUE; +} + static gboolean _ostree_repo_load_cache_summary_file (OstreeRepo *self, const char *filename, @@ -6150,71 +6224,16 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) goto out; - if (gpg_verify_summary) - { - if (summary == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)"); - goto out; - } - - if (signatures == NULL) - { - g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, - "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } - - /* Verify any summary signatures. */ - if (summary != NULL && signatures != NULL) - { - g_autoptr(OstreeGpgVerifyResult) result = NULL; - - result = ostree_repo_verify_summary (self, - name, - summary, - signatures, - cancellable, - error); - if (!ostree_gpg_verify_result_require_valid_signature (result, error)) - goto out; - } - } - if (!_signapi_init_for_remote (self, name, NULL, &signapi_summary_verifiers, error)) goto out; - if (signapi_summary_verifiers) - { - if (summary == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Signature verification enabled, but no summary found (check that the configured URL in remote config is correct)"); - goto out; - } - - if (signatures == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Signature verification enabled, but no summary signatures found (use sign-verify-summary=false in remote config to disable)"); - goto out; - } - - /* Verify any summary signatures. */ - if (summary != NULL && signatures != NULL) - { - g_autoptr(GVariant) sig_variant = NULL; - - sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, - signatures, FALSE); - - if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, NULL, error)) - goto out; - } - } + if (!_ostree_repo_verify_summary (self, name, + gpg_verify_summary, signapi_summary_verifiers, + summary, signatures, + cancellable, error)) + goto out; if (!summary_is_from_cache && summary && signatures) { From f74bc8dd3dd099cb92cdf9c828c48848e0b96703 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:03:14 +0200 Subject: [PATCH 11/58] fetch_summary_with_options: drop unnecessary "goto out" use --- src/libostree/ostree-repo-pull.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ec9983fb..e39249ac 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -6200,7 +6200,6 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GBytes) signatures = NULL; gboolean gpg_verify_summary; g_autoptr(GPtrArray) signapi_summary_verifiers = NULL; - gboolean ret = FALSE; gboolean summary_is_from_cache; g_return_val_if_fail (OSTREE_REPO (self), FALSE); @@ -6208,7 +6207,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (!ostree_repo_get_remote_option (self, name, "metalink", NULL, &metalink_url_string, error)) - goto out; + return FALSE; if (!repo_remote_fetch_summary (self, name, @@ -6219,21 +6218,21 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, &summary_is_from_cache, cancellable, error)) - goto out; + return FALSE; if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) - goto out; + return FALSE; if (!_signapi_init_for_remote (self, name, NULL, &signapi_summary_verifiers, error)) - goto out; + return FALSE; if (!_ostree_repo_verify_summary (self, name, gpg_verify_summary, signapi_summary_verifiers, summary, signatures, cancellable, error)) - goto out; + return FALSE; if (!summary_is_from_cache && summary && signatures) { @@ -6251,7 +6250,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, else { g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; + return FALSE; } } } @@ -6262,10 +6261,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (out_signatures != NULL) *out_signatures = g_steal_pointer (&signatures); - ret = TRUE; - -out: - return ret; + return TRUE; } #else /* HAVE_LIBCURL_OR_LIBSOUP */ From c7df4317bd965006f7d98186b239b6c0618c6217 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:00:35 +0200 Subject: [PATCH 12/58] Add g_autoptr helper for pushing a thread default main context This happens in a bunch of places, and currently each time it does we have to use "goto out" style cleanups, which just isn't looking very nice. --- src/libotutil/otutil.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/libotutil/otutil.h b/src/libotutil/otutil.h index cd312365..7db7270d 100644 --- a/src/libotutil/otutil.h +++ b/src/libotutil/otutil.h @@ -52,6 +52,31 @@ #define ot_journal_print(...) {} #endif +typedef GMainContext GMainContextPopDefault; +static inline void +_ostree_main_context_pop_default_destroy (void *p) +{ + GMainContext *main_context = p; + + if (main_context) + { + g_main_context_pop_thread_default (main_context); + g_main_context_unref (main_context); + } +} + +static inline GMainContextPopDefault * +_ostree_main_context_new_default (void) +{ + GMainContext *main_context = g_main_context_new (); + + g_main_context_push_thread_default (main_context); + return main_context; +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (GMainContextPopDefault, _ostree_main_context_pop_default_destroy) + + #include #include #include From 4b9e712e82d95312a1d26bb6c4ffec17dfcd6bdf Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:10:01 +0200 Subject: [PATCH 13/58] repo_remote_fetch_summary: Use GMainContextPopDefault This allows us to drop the "goto out" use and clean up this function. --- src/libostree/ostree-repo-pull.c | 33 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index e39249ac..cc1ce021 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3095,9 +3095,7 @@ repo_remote_fetch_summary (OstreeRepo *self, GError **error) { g_autoptr(OstreeFetcher) fetcher = NULL; - g_autoptr(GMainContext) mainctx = NULL; - gboolean ret = FALSE; - gboolean from_cache = FALSE; + g_autoptr(GMainContextPopDefault) mainctx = NULL; const char *url_override = NULL; g_autoptr(GVariant) extra_headers = NULL; g_autoptr(GPtrArray) mirrorlist = NULL; @@ -3112,12 +3110,11 @@ repo_remote_fetch_summary (OstreeRepo *self, (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries); } - mainctx = g_main_context_new (); - g_main_context_push_thread_default (mainctx); + mainctx = _ostree_main_context_new_default (); fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, NULL, error); if (fetcher == NULL) - goto out; + return FALSE; if (extra_headers) _ostree_fetcher_set_extra_headers (fetcher, extra_headers); @@ -3132,21 +3129,21 @@ repo_remote_fetch_summary (OstreeRepo *self, else if (url_override) url_string = g_strdup (url_override); else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) - goto out; + return FALSE; if (metalink_url_string == NULL && g_str_has_prefix (url_string, "mirrorlist=")) { if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), n_network_retries, &mirrorlist, cancellable, error)) - goto out; + return FALSE; } else { g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (url_string, error); if (!uri) - goto out; + return FALSE; mirrorlist = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); @@ -3168,7 +3165,7 @@ repo_remote_fetch_summary (OstreeRepo *self, out_signatures, cancellable, error)) - goto out; + return FALSE; if (*out_signatures) { @@ -3178,13 +3175,14 @@ repo_remote_fetch_summary (OstreeRepo *self, out_summary, cancellable, error)) - goto out; + return FALSE; } if (*out_summary) - from_cache = TRUE; + *out_from_cache = TRUE; else { + *out_from_cache = FALSE; if (!_ostree_preload_metadata_file (self, fetcher, mirrorlist, @@ -3194,17 +3192,10 @@ repo_remote_fetch_summary (OstreeRepo *self, out_summary, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - - out: - if (mainctx) - g_main_context_pop_thread_default (mainctx); - - *out_from_cache = from_cache; - return ret; + return TRUE; } /* Create the fetcher by unioning options from the remote config, plus From 3957bff0cb2818bf65adf6db969553d220b6608f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:26:06 +0200 Subject: [PATCH 14/58] Inline repo_remote_fetch_summary This was only used in one place, and (especially with the simplification with GMainContextPopDefault) and the one caller doesn't really do much more than call the helper. Additionally, what little it does (saving the result in the cache) is inherently tied to how the helper work, and will become even more so when we support summary indexes. This is a preparatory cleanup for supporting summary indexes. It doesn't change any behaviour and passes make check on its own. --- src/libostree/ostree-repo-pull.c | 225 ++++++++++++++----------------- 1 file changed, 99 insertions(+), 126 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index cc1ce021..61700e47 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3083,121 +3083,6 @@ fetch_mirrorlist (OstreeFetcher *fetcher, return TRUE; } -static gboolean -repo_remote_fetch_summary (OstreeRepo *self, - const char *name, - const char *metalink_url_string, - GVariant *options, - GBytes **out_summary, - GBytes **out_signatures, - gboolean *out_from_cache, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(OstreeFetcher) fetcher = NULL; - g_autoptr(GMainContextPopDefault) mainctx = NULL; - const char *url_override = NULL; - g_autoptr(GVariant) extra_headers = NULL; - g_autoptr(GPtrArray) mirrorlist = NULL; - const char *append_user_agent = NULL; - guint n_network_retries = DEFAULT_N_NETWORK_RETRIES; - - if (options) - { - (void) g_variant_lookup (options, "override-url", "&s", &url_override); - (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); - (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent); - (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries); - } - - mainctx = _ostree_main_context_new_default (); - - fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, NULL, error); - if (fetcher == NULL) - return FALSE; - - if (extra_headers) - _ostree_fetcher_set_extra_headers (fetcher, extra_headers); - - if (append_user_agent) - _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent); - - { - g_autofree char *url_string = NULL; - if (metalink_url_string) - url_string = g_strdup (metalink_url_string); - else if (url_override) - url_string = g_strdup (url_override); - else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) - return FALSE; - - if (metalink_url_string == NULL && - g_str_has_prefix (url_string, "mirrorlist=")) - { - if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), - n_network_retries, &mirrorlist, cancellable, error)) - return FALSE; - } - else - { - g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (url_string, error); - - if (!uri) - return FALSE; - - mirrorlist = - g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); - g_ptr_array_add (mirrorlist, g_steal_pointer (&uri)); - } - } - - /* FIXME: Send the ETag from the cache with the request for summary.sig to - * avoid downloading summary.sig unnecessarily. This won’t normally provide - * any benefits (but won’t do any harm) since summary.sig is typically 500B - * in size. But if a repository has multiple keys, the signature file will - * grow and this optimisation may be useful. */ - if (!_ostree_preload_metadata_file (self, - fetcher, - mirrorlist, - "summary.sig", - metalink_url_string ? TRUE : FALSE, - n_network_retries, - out_signatures, - cancellable, - error)) - return FALSE; - - if (*out_signatures) - { - if (!_ostree_repo_load_cache_summary_if_same_sig (self, - name, - *out_signatures, - out_summary, - cancellable, - error)) - return FALSE; - } - - if (*out_summary) - *out_from_cache = TRUE; - else - { - *out_from_cache = FALSE; - if (!_ostree_preload_metadata_file (self, - fetcher, - mirrorlist, - "summary", - metalink_url_string ? TRUE : FALSE, - n_network_retries, - out_summary, - cancellable, - error)) - return FALSE; - } - - return TRUE; -} - /* Create the fetcher by unioning options from the remote config, plus * any options specific to this pull (such as extra headers). */ @@ -6191,7 +6076,14 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GBytes) signatures = NULL; gboolean gpg_verify_summary; g_autoptr(GPtrArray) signapi_summary_verifiers = NULL; - gboolean summary_is_from_cache; + gboolean summary_is_from_cache = FALSE; + g_autoptr(OstreeFetcher) fetcher = NULL; + g_autoptr(GMainContextPopDefault) mainctx = NULL; + const char *url_override = NULL; + g_autoptr(GVariant) extra_headers = NULL; + g_autoptr(GPtrArray) mirrorlist = NULL; + const char *append_user_agent = NULL; + guint n_network_retries = DEFAULT_N_NETWORK_RETRIES; g_return_val_if_fail (OSTREE_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); @@ -6200,16 +6092,13 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, &metalink_url_string, error)) return FALSE; - if (!repo_remote_fetch_summary (self, - name, - metalink_url_string, - options, - &summary, - &signatures, - &summary_is_from_cache, - cancellable, - error)) - return FALSE; + if (options) + { + (void) g_variant_lookup (options, "override-url", "&s", &url_override); + (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); + (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent); + (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries); + } if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) return FALSE; @@ -6219,6 +6108,90 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, error)) return FALSE; + mainctx = _ostree_main_context_new_default (); + + fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, NULL, error); + if (fetcher == NULL) + return FALSE; + + if (extra_headers) + _ostree_fetcher_set_extra_headers (fetcher, extra_headers); + + if (append_user_agent) + _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent); + + { + g_autofree char *url_string = NULL; + if (metalink_url_string) + url_string = g_strdup (metalink_url_string); + else if (url_override) + url_string = g_strdup (url_override); + else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) + return FALSE; + + if (metalink_url_string == NULL && + g_str_has_prefix (url_string, "mirrorlist=")) + { + if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), + n_network_retries, &mirrorlist, cancellable, error)) + return FALSE; + } + else + { + g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (url_string, error); + + if (!uri) + return FALSE; + + mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); + g_ptr_array_add (mirrorlist, g_steal_pointer (&uri)); + } + } + + /* FIXME: Send the ETag from the cache with the request for summary.sig to + * avoid downloading summary.sig unnecessarily. This won’t normally provide + * any benefits (but won’t do any harm) since summary.sig is typically 500B + * in size. But if a repository has multiple keys, the signature file will + * grow and this optimisation may be useful. */ + if (!_ostree_preload_metadata_file (self, + fetcher, + mirrorlist, + "summary.sig", + metalink_url_string ? TRUE : FALSE, + n_network_retries, + &signatures, + cancellable, + error)) + return FALSE; + + if (signatures) + { + if (!_ostree_repo_load_cache_summary_if_same_sig (self, + name, + signatures, + &summary, + cancellable, + error)) + return FALSE; + } + + if (summary) + summary_is_from_cache = TRUE; + else + { + if (!_ostree_preload_metadata_file (self, + fetcher, + mirrorlist, + "summary", + metalink_url_string ? TRUE : FALSE, + n_network_retries, + &summary, + cancellable, + error)) + return FALSE; + } + if (!_ostree_repo_verify_summary (self, name, gpg_verify_summary, signapi_summary_verifiers, summary, signatures, From 155b215cd86514c300ba573665dc8320c0c72852 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:34:31 +0200 Subject: [PATCH 15/58] Minor cleanup of _ostree_repo_remote_new_fetcher() Instead of open coding the extra_headers and append_user_agent setting everywhere we do this in the constructor. --- src/libostree/ostree-repo-pull.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 61700e47..21092cd7 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2824,6 +2824,8 @@ static OstreeFetcher * _ostree_repo_remote_new_fetcher (OstreeRepo *self, const char *remote_name, gboolean gzip, + GVariant *extra_headers, + const char *append_user_agent, OstreeFetcherSecurityState *out_state, GError **error) { @@ -2937,6 +2939,12 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, _ostree_fetcher_set_cookie_jar (fetcher, jar_path); } + if (extra_headers) + _ostree_fetcher_set_extra_headers (fetcher, extra_headers); + + if (append_user_agent) + _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent); + success = TRUE; out: @@ -3092,17 +3100,13 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, { g_clear_object (&pull_data->fetcher); pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, FALSE, + pull_data->extra_headers, + pull_data->append_user_agent, &pull_data->fetcher_security_state, error); if (pull_data->fetcher == NULL) return FALSE; - if (pull_data->extra_headers) - _ostree_fetcher_set_extra_headers (pull_data->fetcher, pull_data->extra_headers); - - if (pull_data->append_user_agent) - _ostree_fetcher_set_extra_user_agent (pull_data->fetcher, pull_data->append_user_agent); - return TRUE; } @@ -5454,7 +5458,7 @@ find_remotes_cb (GObject *obj, goto error; fetcher = _ostree_repo_remote_new_fetcher (self, result->remote->name, - TRUE, NULL, &error); + TRUE, NULL, NULL, NULL, &error); if (fetcher == NULL) goto error; @@ -6110,16 +6114,10 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, mainctx = _ostree_main_context_new_default (); - fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, NULL, error); + fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, extra_headers, append_user_agent, NULL, error); if (fetcher == NULL) return FALSE; - if (extra_headers) - _ostree_fetcher_set_extra_headers (fetcher, extra_headers); - - if (append_user_agent) - _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent); - { g_autofree char *url_string = NULL; if (metalink_url_string) From da853a17832b2003faec4bb0a9cc5cfcaf935ef3 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 10 Sep 2020 11:58:38 +0200 Subject: [PATCH 16/58] ostree-repo-pull.c: Extract mirrorlist generation to helper This code was duplicated in 3 places, so move it to a single place to clean things up. --- src/libostree/ostree-repo-pull.c | 140 +++++++++++++++---------------- 1 file changed, 66 insertions(+), 74 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 21092cd7..2a791e59 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3091,6 +3091,46 @@ fetch_mirrorlist (OstreeFetcher *fetcher, return TRUE; } +static gboolean +compute_effective_mirrorlist (OstreeRepo *self, + const char *remote_name_or_baseurl, + const char *url_override, + OstreeFetcher *fetcher, + guint n_network_retries, + GPtrArray **out_mirrorlist, + GCancellable *cancellable, + GError **error) +{ + g_autofree char *baseurl = NULL; + + if (url_override != NULL) + baseurl = g_strdup (url_override); + else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error)) + return FALSE; + + if (g_str_has_prefix (baseurl, "mirrorlist=")) + { + if (!fetch_mirrorlist (fetcher, + baseurl + strlen ("mirrorlist="), + n_network_retries, + out_mirrorlist, + cancellable, error)) + return FALSE; + } + else + { + g_autoptr(OstreeFetcherURI) baseuri = _ostree_fetcher_uri_parse (baseurl, error); + + if (!baseuri) + return FALSE; + + *out_mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); + g_ptr_array_add (*out_mirrorlist, g_steal_pointer (&baseuri)); + } + return TRUE; +} + /* Create the fetcher by unioning options from the remote config, plus * any options specific to this pull (such as extra headers). */ @@ -3618,33 +3658,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!metalink_url_str) { - g_autofree char *baseurl = NULL; - - if (url_override != NULL) - baseurl = g_strdup (url_override); - else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error)) + if (!compute_effective_mirrorlist (self, remote_name_or_baseurl, + url_override, + pull_data->fetcher, + pull_data->n_network_retries, + &pull_data->meta_mirrorlist, + cancellable, error)) goto out; - - if (g_str_has_prefix (baseurl, "mirrorlist=")) - { - if (!fetch_mirrorlist (pull_data->fetcher, - baseurl + strlen ("mirrorlist="), - pull_data->n_network_retries, - &pull_data->meta_mirrorlist, - cancellable, error)) - goto out; - } - else - { - g_autoptr(OstreeFetcherURI) baseuri = _ostree_fetcher_uri_parse (baseurl, error); - - if (!baseuri) - goto out; - - pull_data->meta_mirrorlist = - g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); - g_ptr_array_add (pull_data->meta_mirrorlist, g_steal_pointer (&baseuri)); - } } else { @@ -3703,27 +3723,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else { - if (g_str_has_prefix (contenturl, "mirrorlist=")) - { - if (!fetch_mirrorlist (pull_data->fetcher, - contenturl + strlen ("mirrorlist="), - pull_data->n_network_retries, - &pull_data->content_mirrorlist, - cancellable, error)) - goto out; - } - else - { - g_autoptr(OstreeFetcherURI) contenturi = _ostree_fetcher_uri_parse (contenturl, error); - - if (!contenturi) - goto out; - - pull_data->content_mirrorlist = - g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); - g_ptr_array_add (pull_data->content_mirrorlist, - g_steal_pointer (&contenturi)); - } + if (!compute_effective_mirrorlist (self, remote_name_or_baseurl, + contenturl, + pull_data->fetcher, + pull_data->n_network_retries, + &pull_data->content_mirrorlist, + cancellable, error)) + goto out; } } @@ -6118,34 +6124,20 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (fetcher == NULL) return FALSE; - { - g_autofree char *url_string = NULL; - if (metalink_url_string) - url_string = g_strdup (metalink_url_string); - else if (url_override) - url_string = g_strdup (url_override); - else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) - return FALSE; + if (metalink_url_string) + { + g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (metalink_url_string, error); + if (!uri) + return FALSE; - if (metalink_url_string == NULL && - g_str_has_prefix (url_string, "mirrorlist=")) - { - if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), - n_network_retries, &mirrorlist, cancellable, error)) - return FALSE; - } - else - { - g_autoptr(OstreeFetcherURI) uri = _ostree_fetcher_uri_parse (url_string, error); - - if (!uri) - return FALSE; - - mirrorlist = - g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); - g_ptr_array_add (mirrorlist, g_steal_pointer (&uri)); - } - } + mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); + g_ptr_array_add (mirrorlist, g_steal_pointer (&uri)); + } + else if (!compute_effective_mirrorlist (self, name, url_override, + fetcher, n_network_retries, + &mirrorlist, cancellable, error)) + return FALSE; /* FIXME: Send the ETag from the cache with the request for summary.sig to * avoid downloading summary.sig unnecessarily. This won’t normally provide From b7d1a9746b51fdb4cf14aa173e46e7d165cfe6e4 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 11 Sep 2020 12:14:05 +0200 Subject: [PATCH 17/58] Update the symbols files to match that we're now on 2020.6 --- src/libostree/libostree-devel.sym | 2 +- src/libostree/libostree-released.sym | 5 ++++- tests/test-symbols.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 21669c4a..7f1f7e3e 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,7 +17,7 @@ Boston, MA 02111-1307, USA. ***/ -LIBOSTREE_2020.5 { +LIBOSTREE_2020.6 { global: /* Add symbols here, and uncomment the bits in * Makefile-libostree.am to enable this too. diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index 5c63f78f..aee434cb 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -593,6 +593,9 @@ global: ostree_sysroot_set_mount_namespace_in_use; } LIBOSTREE_2019.6; +/* No new symbols in 2020.2 */ +/* No new symbols in 2020.3 */ + /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2020.4 { global: @@ -615,7 +618,7 @@ global: ostree_sign_summary; } LIBOSTREE_2020.1; -/* No new symbols in 2020.2 */ +/* No new symbols in 2020.5 */ /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 75df37e1..a0781618 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -66,7 +66,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt < Date: Sun, 13 Sep 2020 14:56:06 +0000 Subject: [PATCH 18/58] commit: Tighten scope of two variables Prep for adding `-Wshadow` fixes. --- src/libostree/ostree-repo-commit.c | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 20340257..690075e1 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3943,11 +3943,9 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GFileInfo) child_info = NULL; g_autoptr(GFileInfo) modified_info = NULL; g_autoptr(GVariant) xattrs = NULL; g_autofree guchar *child_file_csum = NULL; - g_autofree char *tmp_checksum = NULL; g_autofree char *relpath = NULL; OstreeRepoCommitFilterResult filter_result; struct stat dir_stbuf; @@ -3955,19 +3953,19 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, if (!glnx_fstat (src_dfd_iter->fd, &dir_stbuf, error)) return FALSE; - child_info = _ostree_stbuf_to_gfileinfo (&dir_stbuf); - - if (modifier != NULL) - { - relpath = ptrarray_path_join (path); - - filter_result = _ostree_repo_commit_modifier_apply (self, modifier, relpath, child_info, &modified_info); - } - else - { - filter_result = OSTREE_REPO_COMMIT_FILTER_ALLOW; - modified_info = g_object_ref (child_info); - } + { + g_autoptr(GFileInfo) child_info = _ostree_stbuf_to_gfileinfo (&dir_stbuf); + if (modifier != NULL) + { + relpath = ptrarray_path_join (path); + filter_result = _ostree_repo_commit_modifier_apply (self, modifier, relpath, child_info, &modified_info); + } + else + { + filter_result = OSTREE_REPO_COMMIT_FILTER_ALLOW; + modified_info = g_object_ref (child_info); + } + } if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW) { @@ -3979,8 +3977,7 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, cancellable, error)) return FALSE; - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (child_file_csum); + g_autofree char *tmp_checksum = ostree_checksum_from_bytes (child_file_csum); ostree_mutable_tree_set_metadata_checksum (mtree, tmp_checksum); } From c4f26bfdc8e1bba7c01016de5af1e8a84a8b38b0 Mon Sep 17 00:00:00 2001 From: Phaedrus Leeds Date: Sat, 12 Sep 2020 14:51:40 -0700 Subject: [PATCH 19/58] Avoid shadowing local variables This should help with code readability. Fixes https://github.com/ostreedev/ostree/issues/2194 --- configure.ac | 1 + src/libostree/ostree-fetcher-curl.c | 6 ++-- .../ostree-repo-finder-avahi-parser.c | 4 +-- src/libostree/ostree-repo-finder-mount.c | 2 +- src/libostree/ostree-repo-finder-override.c | 2 +- src/libostree/ostree-repo-libarchive.c | 6 ++-- src/libostree/ostree-repo-pull.c | 14 ++++---- src/libostree/ostree-repo.c | 18 +++++------ src/libostree/ostree-soup-uri.c | 22 ++++++------- src/libostree/ostree-sysroot-deploy.c | 8 ++--- src/ostree/ot-admin-builtin-set-origin.c | 6 ++-- src/ostree/ot-builtin-checkout.c | 32 +++++++++---------- src/ostree/ot-builtin-commit.c | 6 ++-- src/ostree/ot-builtin-config.c | 8 ++--- src/ostree/ot-builtin-gpg-sign.c | 5 ++- src/ostree/ot-builtin-pull.c | 6 ++-- src/ostree/ot-builtin-sign.c | 6 ++-- src/ostree/ot-remote-builtin-add.c | 3 +- tests/repo-finder-mount.c | 8 ++--- tests/test-repo-finder-config.c | 8 ++--- tests/test-repo-finder-mount.c | 16 +++++----- 21 files changed, 93 insertions(+), 94 deletions(-) diff --git a/configure.ac b/configure.ac index c81507d2..de219eab 100644 --- a/configure.ac +++ b/configure.ac @@ -35,6 +35,7 @@ AS_IF([echo "$CFLAGS" | grep -q -E -e '-Werror($| )'], [], [ CC_CHECK_FLAGS_APPEND([WARN_CFLAGS], [CFLAGS], [\ -pipe \ -Wall \ + -Werror=shadow \ -Werror=empty-body \ -Werror=strict-prototypes \ -Werror=missing-prototypes \ diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index fdf8a2ef..0ce3ff00 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -341,14 +341,14 @@ check_multi_info (OstreeFetcher *fetcher) if (req->idx + 1 == req->mirrorlist->len) { - g_autofree char *msg = g_strdup_printf ("Server returned HTTP %lu", response); + g_autofree char *response_msg = g_strdup_printf ("Server returned HTTP %lu", response); g_task_return_new_error (task, G_IO_ERROR, giocode, - "%s", msg); + "%s", response_msg); if (req->fetcher->remote_name && !((req->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && giocode == G_IO_ERROR_NOT_FOUND)) _ostree_fetcher_journal_failure (req->fetcher->remote_name, - eff_url, msg); + eff_url, response_msg); } else diff --git a/src/libostree/ostree-repo-finder-avahi-parser.c b/src/libostree/ostree-repo-finder-avahi-parser.c index afc9790c..1cf4a11d 100644 --- a/src/libostree/ostree-repo-finder-avahi-parser.c +++ b/src/libostree/ostree-repo-finder-avahi-parser.c @@ -91,14 +91,14 @@ parse_txt_record (const guint8 *txt, /* TODO: Docs. Return value is only valid as long as @txt is. Reference: RFC 6763, §6. */ GHashTable * -_ostree_txt_records_parse (AvahiStringList *txt) +_ostree_txt_records_parse (AvahiStringList *txt_list) { AvahiStringList *l; g_autoptr(GHashTable) out = NULL; out = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_bytes_unref); - for (l = txt; l != NULL; l = avahi_string_list_get_next (l)) + for (l = txt_list; l != NULL; l = avahi_string_list_get_next (l)) { const guint8 *txt; gsize txt_len; diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index c259f3e6..5c8ab1f3 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -336,7 +336,6 @@ ostree_repo_finder_mount_resolve_async (OstreeRepoFinder *finde g_autoptr(GHashTable) repo_to_refs = NULL; /* (element-type UriAndKeyring GHashTable) */ GHashTable *supported_ref_to_checksum; /* (element-type OstreeCollectionRef utf8) */ GHashTableIter iter; - UriAndKeyring *repo; g_autoptr(GError) local_error = NULL; mount_name = g_mount_get_name (mount); @@ -525,6 +524,7 @@ G_GNUC_END_IGNORE_DEPRECATIONS /* Aggregate the results. */ g_hash_table_iter_init (&iter, repo_to_refs); + UriAndKeyring *repo; while (g_hash_table_iter_next (&iter, (gpointer *) &repo, (gpointer *) &supported_ref_to_checksum)) { g_autoptr(OstreeRemote) remote = NULL; diff --git a/src/libostree/ostree-repo-finder-override.c b/src/libostree/ostree-repo-finder-override.c index 32199546..d6fb5a9b 100644 --- a/src/libostree/ostree-repo-finder-override.c +++ b/src/libostree/ostree-repo-finder-override.c @@ -151,7 +151,6 @@ ostree_repo_finder_override_resolve_async (OstreeRepoFinder *fi GHashTable *supported_ref_to_checksum; /* (element-type OstreeCollectionRef utf8) */ GHashTableIter iter; const gchar *remote_uri; - OstreeRemote *remote; task = g_task_new (finder, cancellable, callback, user_data); g_task_set_source_tag (task, ostree_repo_finder_override_resolve_async); @@ -242,6 +241,7 @@ ostree_repo_finder_override_resolve_async (OstreeRepoFinder *fi /* Aggregate the results. */ g_hash_table_iter_init (&iter, repo_remote_to_refs); + OstreeRemote *remote; while (g_hash_table_iter_next (&iter, (gpointer *) &remote, (gpointer *) &supported_ref_to_checksum)) g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, NULL, 0)); diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index d55459f4..ef7252e8 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -1161,16 +1161,16 @@ write_directory_to_libarchive_recurse (OstreeRepo *self, { guint8 buf[8192]; g_autoptr(GInputStream) file_in = NULL; - g_autoptr(GFileInfo) file_info = NULL; + g_autoptr(GFileInfo) regular_file_info = NULL; const char *checksum; checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path); - if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL, + if (!ostree_repo_load_file (self, checksum, &file_in, ®ular_file_info, NULL, cancellable, error)) goto out; - archive_entry_set_size (entry, g_file_info_get_size (file_info)); + archive_entry_set_size (entry, g_file_info_get_size (regular_file_info)); if (archive_write_header (a, entry) != ARCHIVE_OK) { diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 2a791e59..a6401907 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1625,13 +1625,13 @@ scan_commit_object (OtPullData *pull_data, } if (pull_data->timestamp_check_from_rev) { - g_autoptr(GVariant) commit = NULL; + g_autoptr(GVariant) timestamp_commit = NULL; if (!ostree_repo_load_commit (pull_data->repo, pull_data->timestamp_check_from_rev, - &commit, NULL, error)) + ×tamp_commit, NULL, error)) return glnx_prefix_error (error, "Reading %s for timestamp-check-from-rev", pull_data->timestamp_check_from_rev); - guint64 ts = ostree_commit_get_timestamp (commit); + guint64 ts = ostree_commit_get_timestamp (timestamp_commit); if (!_ostree_compare_timestamps (pull_data->timestamp_check_from_rev, ts, checksum, new_ts, error)) return FALSE; } @@ -3389,7 +3389,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *url_override = NULL; gboolean inherit_transaction = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ - int i; + gsize i; g_autofree char **opt_localcache_repos = NULL; g_autoptr(GVariantIter) ref_keyring_map_iter = NULL; g_autoptr(GVariant) summary_bytes_v = NULL; @@ -3849,7 +3849,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { g_autoptr(GBytes) bytes_sig = NULL; - gsize i, n; + gsize n; g_autoptr(GVariant) refs = NULL; g_autoptr(GVariant) deltas = NULL; g_autoptr(GVariant) additional_metadata = NULL; @@ -5227,7 +5227,7 @@ find_remotes_process_refs (OstreeRepo *self, static void find_remotes_cb (GObject *obj, - GAsyncResult *result, + GAsyncResult *async_result, gpointer user_data) { OstreeRepo *self; @@ -5259,7 +5259,7 @@ find_remotes_cb (GObject *obj, /* progress = data->progress; */ /* Finish finding the remotes. */ - results = ostree_repo_finder_resolve_all_finish (result, &error); + results = ostree_repo_finder_resolve_all_finish (async_result, &error); if (results == NULL) { diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 621668d7..4283f68e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3782,14 +3782,14 @@ load_metadata_internal (OstreeRepo *self, g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (sha256); *out_state = 0; - glnx_autofd int fd = -1; - if (!ot_openat_ignore_enoent (self->repo_dir_fd, commitpartial_path, &fd, error)) + glnx_autofd int commitpartial_fd = -1; + if (!ot_openat_ignore_enoent (self->repo_dir_fd, commitpartial_path, &commitpartial_fd, error)) return FALSE; - if (fd != -1) + if (commitpartial_fd != -1) { *out_state |= OSTREE_REPO_COMMIT_STATE_PARTIAL; char reason; - if (read (fd, &reason, 1) == 1) + if (read (commitpartial_fd, &reason, 1) == 1) { if (reason == 'f') *out_state |= OSTREE_REPO_COMMIT_STATE_FSCK_PARTIAL; @@ -5831,19 +5831,19 @@ ostree_repo_regenerate_summary (OstreeRepo *self, collection_map = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) g_hash_table_unref); - const OstreeCollectionRef *ref; + const OstreeCollectionRef *c_ref; const char *checksum; - while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + while (g_hash_table_iter_next (&iter, (gpointer *) &c_ref, (gpointer *) &checksum)) { - GHashTable *ref_map = g_hash_table_lookup (collection_map, ref->collection_id); + GHashTable *ref_map = g_hash_table_lookup (collection_map, c_ref->collection_id); if (ref_map == NULL) { ref_map = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); - g_hash_table_insert (collection_map, ref->collection_id, ref_map); + g_hash_table_insert (collection_map, c_ref->collection_id, ref_map); } - g_hash_table_insert (ref_map, ref->ref_name, (gpointer) checksum); + g_hash_table_insert (ref_map, c_ref->ref_name, (gpointer) checksum); } g_autoptr(GVariantBuilder) collection_refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); diff --git a/src/libostree/ostree-soup-uri.c b/src/libostree/ostree-soup-uri.c index a3fa2acc..ba14e5c6 100644 --- a/src/libostree/ostree-soup-uri.c +++ b/src/libostree/ostree-soup-uri.c @@ -348,7 +348,7 @@ soup_uri_new_with_base (SoupURI *base, const char *uri_string) { SoupURI *uri, fixed_base; const char *end, *hash, *colon, *at, *path, *question; - const char *p, *hostend; + const char *c, *hostend; gboolean remove_dot_segments = TRUE; int len; @@ -402,17 +402,17 @@ soup_uri_new_with_base (SoupURI *base, const char *uri_string) } /* Find scheme */ - p = uri_string; - while (p < end && (g_ascii_isalpha (*p) || - (p > uri_string && (g_ascii_isdigit (*p) || - *p == '.' || - *p == '+' || - *p == '-')))) - p++; + c = uri_string; + while (c < end && (g_ascii_isalpha (*c) || + (c > uri_string && (g_ascii_isdigit (*c) || + *c == '.' || + *c == '+' || + *c == '-')))) + c++; - if (p > uri_string && *p == ':') { - uri->scheme = soup_uri_parse_scheme (uri_string, p - uri_string); - uri_string = p + 1; + if (c > uri_string && *c == ':') { + uri->scheme = soup_uri_parse_scheme (uri_string, c - uri_string); + uri_string = c + 1; } if (uri_string == end && !base && !uri->fragment) { diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 65d54b46..3f337a50 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1945,8 +1945,8 @@ install_deployment_kernel (OstreeSysroot *sysroot, if (kernel_layout->devicetree_namever) { - g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_namever, NULL); - ostree_bootconfig_parser_set (bootconfig, "devicetree", boot_relpath); + g_autofree char * dt_boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_namever, NULL); + ostree_bootconfig_parser_set (bootconfig, "devicetree", dt_boot_relpath); } else if (kernel_layout->devicetree_srcpath) { @@ -1954,8 +1954,8 @@ install_deployment_kernel (OstreeSysroot *sysroot, * want to point to a whole directory of device trees. * See: https://github.com/ostreedev/ostree/issues/1900 */ - g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_srcpath, NULL); - ostree_bootconfig_parser_set (bootconfig, "fdtdir", boot_relpath); + g_autofree char * dt_boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_srcpath, NULL); + ostree_bootconfig_parser_set (bootconfig, "fdtdir", dt_boot_relpath); } /* Note this is parsed in ostree-impl-system-generator.c */ diff --git a/src/ostree/ot-admin-builtin-set-origin.c b/src/ostree/ot-admin-builtin-set-origin.c index 9d96512e..4dc68a00 100644 --- a/src/ostree/ot-admin-builtin-set-origin.c +++ b/src/ostree/ot-admin-builtin-set-origin.c @@ -95,7 +95,7 @@ ot_admin_builtin_set_origin (int argc, char **argv, OstreeCommandInvocation *inv { char **iter; g_autoptr(GVariantBuilder) optbuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_autoptr(GVariant) options = NULL; + g_autoptr(GVariant) remote_options = NULL; for (iter = opt_set; iter && *iter; iter++) { @@ -110,12 +110,12 @@ ot_admin_builtin_set_origin (int argc, char **argv, OstreeCommandInvocation *inv subkey, g_variant_new_variant (g_variant_new_string (subvalue))); } - options = g_variant_ref_sink (g_variant_builder_end (optbuilder)); + remote_options = g_variant_ref_sink (g_variant_builder_end (optbuilder)); if (!ostree_repo_remote_change (repo, NULL, OSTREE_REPO_REMOTE_CHANGE_ADD_IF_NOT_EXISTS, remotename, url, - options, + remote_options, cancellable, error)) goto out; } diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 813dbb9e..adb763a9 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -135,14 +135,14 @@ process_one_checkout (OstreeRepo *repo, opt_bareuseronly_dirs || opt_union_identical || opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix) { - OstreeRepoCheckoutAtOptions options = { 0, }; + OstreeRepoCheckoutAtOptions checkout_options = { 0, }; /* do this early so option checking also catches force copy conflicts */ if (opt_selinux_policy) opt_force_copy = TRUE; if (opt_user_mode) - options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; + checkout_options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; /* Can't union these */ if (opt_union && opt_union_add) { @@ -173,9 +173,9 @@ process_one_checkout (OstreeRepo *repo, goto out; } else if (opt_union) - options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + checkout_options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; else if (opt_union_add) - options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; + checkout_options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; else if (opt_union_identical) { if (!opt_require_hardlinks) @@ -184,12 +184,12 @@ process_one_checkout (OstreeRepo *repo, "--union-identical requires --require-hardlinks"); goto out; } - options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL; + checkout_options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL; } if (opt_whiteouts) - options.process_whiteouts = TRUE; + checkout_options.process_whiteouts = TRUE; if (subpath) - options.subpath = subpath; + checkout_options.subpath = subpath; g_autoptr(OstreeSePolicy) policy = NULL; if (opt_selinux_policy) @@ -203,8 +203,8 @@ process_one_checkout (OstreeRepo *repo, policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error); if (!policy) goto out; - options.sepolicy = policy; - options.sepolicy_prefix = opt_selinux_prefix; + checkout_options.sepolicy = policy; + checkout_options.sepolicy_prefix = opt_selinux_prefix; } g_autoptr(GHashTable) skip_list = @@ -214,16 +214,16 @@ process_one_checkout (OstreeRepo *repo, if (!ot_parse_file_by_line (opt_skiplist_file, handle_skiplist_line, skip_list, cancellable, error)) goto out; - options.filter = checkout_filter; - options.filter_user_data = skip_list; + checkout_options.filter = checkout_filter; + checkout_options.filter_user_data = skip_list; } - options.no_copy_fallback = opt_require_hardlinks; - options.force_copy = opt_force_copy; - options.force_copy_zerosized = opt_force_copy_zerosized; - options.bareuseronly_dirs = opt_bareuseronly_dirs; + checkout_options.no_copy_fallback = opt_require_hardlinks; + checkout_options.force_copy = opt_force_copy; + checkout_options.force_copy_zerosized = opt_force_copy_zerosized; + checkout_options.bareuseronly_dirs = opt_bareuseronly_dirs; - if (!ostree_repo_checkout_at (repo, &options, + if (!ostree_repo_checkout_at (repo, &checkout_options, AT_FDCWD, destination, resolved_commit, cancellable, error)) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index e2fcf103..48fa2928 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -614,10 +614,10 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio if (opt_base) { g_autofree char *base_commit = NULL; - g_autoptr(GFile) root = NULL; - if (!ostree_repo_read_commit (repo, opt_base, &root, &base_commit, cancellable, error)) + g_autoptr(GFile) base_root = NULL; + if (!ostree_repo_read_commit (repo, opt_base, &base_root, &base_commit, cancellable, error)) goto out; - OstreeRepoFile *rootf = (OstreeRepoFile*) root; + OstreeRepoFile *rootf = (OstreeRepoFile*) base_root; mtree = ostree_mutable_tree_new_from_checksum (repo, ostree_repo_file_tree_get_contents_checksum (rootf), diff --git a/src/ostree/ot-builtin-config.c b/src/ostree/ot-builtin-config.c index 811a8381..64e434aa 100644 --- a/src/ostree/ot-builtin-config.c +++ b/src/ostree/ot-builtin-config.c @@ -134,7 +134,7 @@ ostree_builtin_config (int argc, char **argv, OstreeCommandInvocation *invocatio else if (!strcmp (op, "get")) { GKeyFile *readonly_config = NULL; - g_autofree char *value = NULL; + g_autofree char *read_value = NULL; if (opt_group) { if (argc < 3) @@ -160,11 +160,11 @@ ostree_builtin_config (int argc, char **argv, OstreeCommandInvocation *invocatio } readonly_config = ostree_repo_get_config (repo); - value = g_key_file_get_string (readonly_config, section, key, error); - if (value == NULL) + read_value = g_key_file_get_string (readonly_config, section, key, error); + if (read_value == NULL) return FALSE; - g_print ("%s\n", value); + g_print ("%s\n", read_value); } else if (!strcmp (op, "unset")) { diff --git a/src/ostree/ot-builtin-gpg-sign.c b/src/ostree/ot-builtin-gpg-sign.c index 6babbf22..bde9180a 100644 --- a/src/ostree/ot-builtin-gpg-sign.c +++ b/src/ostree/ot-builtin-gpg-sign.c @@ -171,9 +171,8 @@ delete_signatures (OstreeRepo *repo, while (!g_queue_is_empty (&signatures)) { - GVariant *child = g_queue_pop_head (&signatures); - g_variant_builder_add_value (&signature_builder, child); - g_variant_unref (child); + g_autoptr(GVariant) sigchild = g_queue_pop_head (&signatures); + g_variant_builder_add_value (&signature_builder, sigchild); } g_variant_dict_insert_value (&metadata_dict, diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index e69d62e3..ed0ec556 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -271,7 +271,7 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, { GVariantBuilder builder; - g_autoptr(GVariant) options = NULL; + g_autoptr(GVariant) pull_options = NULL; g_auto(GLnxConsoleRef) console = { 0, }; g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); @@ -378,9 +378,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, #endif /* OSTREE_DISABLE_GPGME */ } - options = g_variant_ref_sink (g_variant_builder_end (&builder)); + pull_options = g_variant_ref_sink (g_variant_builder_end (&builder)); - if (!ostree_repo_pull_with_options (repo, remote, options, + if (!ostree_repo_pull_with_options (repo, remote, pull_options, progress, cancellable, error)) goto out; diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index c7777489..2f90acd1 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -167,7 +167,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, if ((n_key_ids == 0) || opt_filename) { g_autoptr (GVariantBuilder) builder = NULL; - g_autoptr (GVariant) options = NULL; + g_autoptr (GVariant) sign_options = NULL; builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); /* Use custom directory with public and revoked keys instead of system-wide directories */ @@ -176,9 +176,9 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, /* The last chance for verification source -- system files */ if (opt_filename) g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); - options = g_variant_builder_end (builder); + sign_options = g_variant_builder_end (builder); - if (!ostree_sign_load_pk (sign, options, error)) + if (!ostree_sign_load_pk (sign, sign_options, error)) goto out; if (ostree_sign_commit_verify (sign, diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 172625d2..61539ec1 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -104,7 +104,6 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio g_autoptr(GString) sign_verify = NULL; const char *remote_name; const char *remote_url; - char **iter; g_autoptr(GVariantBuilder) optbuilder = NULL; g_autoptr(GVariant) options = NULL; gboolean ret = FALSE; @@ -161,7 +160,7 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio g_variant_builder_add (optbuilder, "{s@v}", "contenturl", g_variant_new_variant (g_variant_new_string (opt_contenturl))); - for (iter = opt_set; iter && *iter; iter++) + for (char **iter = opt_set; iter && *iter; iter++) { const char *keyvalue = *iter; g_autofree char *subkey = NULL; diff --git a/tests/repo-finder-mount.c b/tests/repo-finder-mount.c index 2cb1d230..3d068af7 100644 --- a/tests/repo-finder-mount.c +++ b/tests/repo-finder-mount.c @@ -93,16 +93,16 @@ main (int argc, char **argv) g_ptr_array_add (refs, NULL); /* NULL terminator */ - g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GAsyncResult) async_result = NULL; ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), (const OstreeCollectionRef * const *) refs->pdata, - parent_repo, NULL, result_cb, &result); + parent_repo, NULL, result_cb, &async_result); - while (result == NULL) + while (async_result == NULL) g_main_context_iteration (context, TRUE); g_autoptr(GPtrArray) results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), - result, &error); + async_result, &error); g_assert_no_error (error); /* Check that the results are correct: the invalid refs should have been diff --git a/tests/test-repo-finder-config.c b/tests/test-repo-finder-config.c index ead9e907..b3e8a264 100644 --- a/tests/test-repo-finder-config.c +++ b/tests/test-repo-finder-config.c @@ -230,7 +230,7 @@ test_repo_finder_config_mixed_configs (Fixture *fixture, { g_autoptr(OstreeRepoFinderConfig) finder = NULL; g_autoptr(GMainContext) context = NULL; - g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GAsyncResult) async_result = NULL; g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ g_autoptr(GError) error = NULL; gsize i; @@ -266,13 +266,13 @@ test_repo_finder_config_mixed_configs (Fixture *fixture, /* Resolve the refs. */ ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, - fixture->parent_repo, NULL, result_cb, &result); + fixture->parent_repo, NULL, result_cb, &async_result); - while (result == NULL) + while (async_result == NULL) g_main_context_iteration (context, TRUE); results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), - result, &error); + async_result, &error); g_assert_no_error (error); g_assert_nonnull (results); g_assert_cmpuint (results->len, ==, 3); diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c index e4130921..45e58fa4 100644 --- a/tests/test-repo-finder-mount.c +++ b/tests/test-repo-finder-mount.c @@ -317,7 +317,7 @@ test_repo_finder_mount_mixed_mounts (Fixture *fixture, g_autoptr(OstreeRepoFinderMount) finder = NULL; g_autoptr(GVolumeMonitor) monitor = NULL; g_autoptr(GMainContext) context = NULL; - g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GAsyncResult) async_result = NULL; g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ g_autoptr(GError) error = NULL; g_autoptr(GList) mounts = NULL; /* (element-type OstreeMockMount) */ @@ -395,13 +395,13 @@ test_repo_finder_mount_mixed_mounts (Fixture *fixture, /* Resolve the refs. */ ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, fixture->parent_repo, - NULL, result_cb, &result); + NULL, result_cb, &async_result); - while (result == NULL) + while (async_result == NULL) g_main_context_iteration (context, TRUE); results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), - result, &error); + async_result, &error); g_assert_no_error (error); g_assert_nonnull (results); g_assert_cmpuint (results->len, ==, 4); @@ -466,7 +466,7 @@ test_repo_finder_mount_well_known (Fixture *fixture, g_autoptr(OstreeRepoFinderMount) finder = NULL; g_autoptr(GVolumeMonitor) monitor = NULL; g_autoptr(GMainContext) context = NULL; - g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GAsyncResult) async_result = NULL; g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ g_autoptr(GError) error = NULL; g_autoptr(GList) mounts = NULL; /* (element-type OstreeMockMount) */ @@ -507,13 +507,13 @@ test_repo_finder_mount_well_known (Fixture *fixture, /* Resolve the refs. */ ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, fixture->parent_repo, - NULL, result_cb, &result); + NULL, result_cb, &async_result); - while (result == NULL) + while (async_result == NULL) g_main_context_iteration (context, TRUE); results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), - result, &error); + async_result, &error); g_assert_no_error (error); g_assert_nonnull (results); g_assert_cmpuint (results->len, ==, 2); From 46667567c5a1f17ecb2efbf8fd21e7711539003e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Wed, 8 Jul 2020 12:15:15 +0200 Subject: [PATCH 20/58] lib/deltas: Add inline signature for static-delta superblock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the commits contained in the single static-delta file are signed so we can check them and operate on trusted data, the superblock isn't signed in any way, so it end up operating on untrusted data to: 1. actually find where the trusted data is, and 2. check whether the update is fit for the current device by looking at the collection id stored in the metadata This commit generates signatures of all static data, and concatenate them to the existing static delta format, i.e. as a GVariant layout `a{sv}ay` where - a{sv}: signatures - ay: existing delta variant Signed-off-by: Frédéric Danis --- .../ostree-repo-static-delta-compilation.c | 97 +++++++++++++++++-- .../ostree-repo-static-delta-private.h | 19 ++++ 2 files changed, 110 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 3e9a8e30..753f8aee 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -36,6 +36,8 @@ #include "libglnx.h" #include "ostree-varint.h" #include "bsdiff/bsdiff.h" +#include "ostree-autocleanups.h" +#include "ostree-sign.h" #define CONTENT_SIZE_SIMILARITY_THRESHOLD_PERCENT (30) @@ -1335,6 +1337,8 @@ get_fallback_headers (OstreeRepo *self, * - verbose: b: Print diagnostic messages. Default FALSE. * - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN) * - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository. + * - sign-name: ay: Signature type to use. + * - sign-key-ids: as: Array of keys used to sign delta superblock. */ gboolean ostree_repo_static_delta_generate (OstreeRepo *self, @@ -1368,6 +1372,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GPtrArray) builder_fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); g_auto(GLnxTmpfile) descriptor_tmpf = { 0, }; g_autoptr(OtVariantBuilder) descriptor_builder = NULL; + const char *opt_sign_name; + const char **opt_key_ids; if (!g_variant_lookup (params, "min-fallback-size", "u", &min_fallback_size)) min_fallback_size = 4; @@ -1407,6 +1413,12 @@ ostree_repo_static_delta_generate (OstreeRepo *self, if (!g_variant_lookup (params, "filename", "^&ay", &opt_filename)) opt_filename = NULL; + if (!g_variant_lookup (params, "sign-name", "^&ay", &opt_sign_name)) + opt_sign_name = NULL; + + if (!g_variant_lookup (params, "sign-key-ids", "^a&s", &opt_key_ids)) + opt_key_ids = NULL; + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, to, &to_commit, error)) return FALSE; @@ -1442,7 +1454,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, cancellable, error)) return FALSE; - if (!glnx_open_tmpfile_linkable_at (descriptor_dfd, ".", O_WRONLY | O_CLOEXEC, + if (!glnx_open_tmpfile_linkable_at (descriptor_dfd, ".", O_RDWR | O_CLOEXEC, &descriptor_tmpf, error)) return FALSE; @@ -1586,12 +1598,85 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_printerr ("bsdiff=%u objects\n", builder.n_bsdiff); } - if (fchmod (descriptor_tmpf.fd, 0644) < 0) - return glnx_throw_errno_prefix (error, "fchmod"); + if (opt_sign_name != NULL && opt_key_ids != NULL) + { + g_autoptr(GBytes) tmpdata = NULL; + g_autoptr(OstreeSign) sign = NULL; + const gchar *signature_key = NULL; + g_autoptr(GVariantBuilder) signature_builder = NULL; + g_auto(GLnxTmpfile) descriptor_sign_tmpf = { 0, }; + g_autoptr(OtVariantBuilder) descriptor_sign_builder = NULL; - if (!glnx_link_tmpfile_at (&descriptor_tmpf, GLNX_LINK_TMPFILE_REPLACE, - descriptor_dfd, descriptor_name, error)) - return FALSE; + lseek (descriptor_tmpf.fd, 0, SEEK_SET); + tmpdata = glnx_fd_readall_bytes (descriptor_tmpf.fd, cancellable, error); + if (!tmpdata) + return FALSE; + + sign = ostree_sign_get_by_name (opt_sign_name, error); + if (sign == NULL) + return FALSE; + + signature_key = ostree_sign_metadata_key (sign); + const gchar *signature_format = ostree_sign_metadata_format (sign); + + signature_builder = g_variant_builder_new (G_VARIANT_TYPE (signature_format)); + + for (const char **iter = opt_key_ids; iter && *iter; iter++) + { + const char *keyid = *iter; + g_autoptr(GVariant) secret_key = NULL; + g_autoptr(GBytes) signature_bytes = NULL; + + secret_key = g_variant_new_string (keyid); + if (!ostree_sign_set_sk (sign, secret_key, error)) + return FALSE; + + if (!ostree_sign_data (sign, tmpdata, &signature_bytes, + NULL, error)) + return FALSE; + + g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes)); + } + + if (!glnx_open_tmpfile_linkable_at (descriptor_dfd, ".", O_WRONLY | O_CLOEXEC, + &descriptor_sign_tmpf, error)) + return FALSE; + + descriptor_sign_builder = ot_variant_builder_new (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SIGNED_FORMAT), + descriptor_sign_tmpf.fd); + + if (!ot_variant_builder_add (descriptor_sign_builder, error, "t", + GUINT64_TO_BE (OSTREE_STATIC_DELTA_SIGNED_MAGIC))) + return FALSE; + if (!ot_variant_builder_add (descriptor_sign_builder, error, "@ay", ot_gvariant_new_ay_bytes (tmpdata))) + return FALSE; + if (!ot_variant_builder_open (descriptor_sign_builder, G_VARIANT_TYPE ("a{sv}"), error)) + return FALSE; + if (!ot_variant_builder_add (descriptor_sign_builder, error, "{sv}", + signature_key, g_variant_builder_end(signature_builder))) + return FALSE; + if (!ot_variant_builder_close (descriptor_sign_builder, error)) + return FALSE; + + if (!ot_variant_builder_end (descriptor_sign_builder, error)) + return FALSE; + + if (fchmod (descriptor_sign_tmpf.fd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + + if (!glnx_link_tmpfile_at (&descriptor_sign_tmpf, GLNX_LINK_TMPFILE_REPLACE, + descriptor_dfd, descriptor_name, error)) + return FALSE; + } + else + { + if (fchmod (descriptor_tmpf.fd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + + if (!glnx_link_tmpfile_at (&descriptor_tmpf, GLNX_LINK_TMPFILE_REPLACE, + descriptor_dfd, descriptor_name, error)) + return FALSE; + } return TRUE; } diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index ff8de9d4..5a2e6879 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -104,6 +104,25 @@ G_BEGIN_DECLS */ #define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")" +/** + * OSTREE_STATIC_DELTA_SIGNED_FORMAT + * + * magic: t magic number, 8 bytes for alignment + * superblock: ay delta supeblock variant + * signatures: a{sv} + * + * The signed static delta starts with the 'OSTSGNDT' magic number followed by + * the array of bytes containing the superblock used for the signature. + * + * Then, the signatures array contains the signatures of the superblock. A + * signature has the following form: + * type: signature key + * signature: variant depending on type used + */ +#define OSTREE_STATIC_DELTA_SIGNED_FORMAT "(taya{sv})" + +#define OSTREE_STATIC_DELTA_SIGNED_MAGIC 0x4F535453474E4454 /* OSTSGNDT */ + typedef enum { OSTREE_STATIC_DELTA_OPEN_FLAGS_NONE = 0, OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM = (1 << 0), From 92efbc00d80f074bf24605ec10fafac9e7eee697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Tue, 26 Nov 2019 11:20:10 +0100 Subject: [PATCH 21/58] bin/static-delta: Add support to sign superblock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add signing ability to "static-delta generate" builtin. Signed-off-by: Frédéric Danis --- bash/ostree | 5 +++ man/ostree-static-delta.xml | 33 +++++++++++++++ src/ostree/ot-builtin-static-delta.c | 62 ++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/bash/ostree b/bash/ostree index 7256e40a..a7389bd7 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1613,6 +1613,8 @@ _ostree_static_delta_generate() { --repo --set-endianness --to + --sign + --sign-type " local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) @@ -1630,6 +1632,9 @@ _ostree_static_delta_generate() { COMPREPLY=( $( compgen -W "l B" -- "$cur" ) ) return 0 ;; + $options_with_args_glob ) + return 0 + ;; esac case "$cur" in diff --git a/man/ostree-static-delta.xml b/man/ostree-static-delta.xml index dfeef28b..a4bef237 100644 --- a/man/ostree-static-delta.xml +++ b/man/ostree-static-delta.xml @@ -113,6 +113,39 @@ Boston, MA 02111-1307, USA. + + =ENGINE + + + Use particular signature engine. Currently + available ed25519 and dummy + signature types. + + The default is ed25519. + + + + + ="KEY-ID" + + There KEY-ID is: + + + + + base64-encoded secret key for signing. + + + + + + + ASCII-string used as secret key. + + + + + diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 4f9ff2b2..d5e93783 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -40,6 +40,9 @@ static gboolean opt_swap_endianness; static gboolean opt_inline; static gboolean opt_disable_bsdiff; static gboolean opt_if_not_exists; +static char **opt_key_ids; +static char *opt_sign_name; +static char *opt_keysfilename; #define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) @@ -88,6 +91,11 @@ static GOptionEntry generate_options[] = { { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, { "filename", 0, 0, G_OPTION_ARG_FILENAME, &opt_filename, "Write the delta content to PATH (a directory). If not specified, the OSTree repository is used", "PATH"}, + { "sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "Sign the delta with", "KEY_ID"}, + { "sign-type", 0, 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, +#if defined(HAVE_LIBSODIUM) + { "keys-file", 0, 0, G_OPTION_ARG_STRING, &opt_keysfilename, "Read key(s) from file", "NAME"}, +#endif { NULL } }; @@ -326,6 +334,60 @@ ot_static_delta_builtin_generate (int argc, char **argv, OstreeCommandInvocation if (opt_endianness || opt_swap_endianness) g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness)); + if (opt_key_ids || opt_keysfilename) + { + g_autoptr(GPtrArray) key_ids = g_ptr_array_new (); + + for (char **iter = opt_key_ids; iter != NULL && *iter != NULL; ++iter) + g_ptr_array_add (key_ids, *iter); + + if (opt_keysfilename) + { + g_autoptr (GFile) keyfile = NULL; + g_autoptr (GFileInputStream) key_stream_in = NULL; + g_autoptr (GDataInputStream) key_data_in = NULL; + + if (!g_file_test (opt_keysfilename, G_FILE_TEST_IS_REGULAR)) + { + g_warning ("Can't open file '%s' with keys", opt_keysfilename); + return glnx_throw (error, "File object '%s' is not a regular file", opt_keysfilename); + } + + keyfile = g_file_new_for_path (opt_keysfilename); + key_stream_in = g_file_read (keyfile, NULL, error); + if (key_stream_in == NULL) + return FALSE; + + key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); + g_assert (key_data_in != NULL); + + /* Use simple file format with just a list of base64 public keys per line */ + while (TRUE) + { + gsize len = 0; + g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); + g_autoptr (GVariant) sk = NULL; + + if (*error != NULL) + return FALSE; + + if (line == NULL) + break; + + // Pass the key as a string + g_ptr_array_add (key_ids, g_strdup (line)); + } + } + + g_autoptr(GVariant) key_ids_v = g_variant_new_strv ((const char *const *)key_ids->pdata, + key_ids->len); + g_variant_builder_add (parambuilder, "{s@v}", "sign-key-ids", + g_variant_new_variant (g_steal_pointer (&key_ids_v))); + } + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; + g_variant_builder_add (parambuilder, "{sv}", "sign-name", + g_variant_new_bytestring (opt_sign_name)); + g_print ("Generating static delta:\n"); g_print (" From: %s\n", from_resolved ? from_resolved : "empty"); g_print (" To: %s\n", to_resolved); From 02a19b2c96d9f84d23dcd15b3f20e507f6573a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Thu, 28 Nov 2019 12:18:59 +0100 Subject: [PATCH 22/58] lib/deltas: Add signature check API for static-delta superblock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This retrieves the signatures and pass the static delta block as an array of bytes to ostree_sign_data_verify(). Signed-off-by: Frédéric Danis --- Makefile-libostree.am | 2 +- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-static-delta-core.c | 128 ++++++++++++++++++ src/libostree/ostree-repo.h | 8 ++ 5 files changed, 139 insertions(+), 1 deletion(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a180e86b..1d31c4d8 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -183,7 +183,7 @@ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym #if BUILDOPT_IS_DEVEL_BUILD -#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym #endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 979c8e93..8cafeb1f 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -413,6 +413,7 @@ ostree_repo_list_static_delta_names OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate ostree_repo_static_delta_execute_offline +ostree_repo_static_delta_verify_signature ostree_repo_traverse_new_reachable ostree_repo_traverse_new_parents ostree_repo_traverse_parents_get_commits diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 7f1f7e3e..9a847b92 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -22,6 +22,7 @@ global: /* Add symbols here, and uncomment the bits in * Makefile-libostree.am to enable this too. */ + ostree_repo_static_delta_verify_signature; } LIBOSTREE_2020.4; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 835ec7f3..857473db 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -210,6 +210,61 @@ _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, return TRUE; } +static gboolean +_ostree_repo_static_delta_is_signed (OstreeRepo *self, + int fd, + GPtrArray **out_value, + GError **error) +{ + g_autoptr(GVariant) delta = NULL; + g_autoptr(GVariant) delta_sign_magic = NULL; + g_autoptr(GVariant) delta_sign = NULL; + GVariantIter iter; + GVariant *item; + g_autoptr(GPtrArray) signatures = NULL; + gboolean ret = FALSE; + + if (out_value) + *out_value = NULL; + + if (!ot_variant_read_fd (fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, TRUE, &delta, error)) + return FALSE; + + delta_sign_magic = g_variant_get_child_value (delta, 0); + if (delta_sign_magic == NULL) + return glnx_throw (error, "no signatures in static-delta"); + + if (GUINT64_FROM_BE (g_variant_get_uint64 (delta_sign_magic)) != OSTREE_STATIC_DELTA_SIGNED_MAGIC) + return glnx_throw (error, "no signatures in static-delta"); + + delta_sign = g_variant_get_child_value (delta, 2); + if (delta_sign == NULL) + return glnx_throw (error, "no signatures in static-delta"); + + if (out_value) + signatures = g_ptr_array_new_with_free_func (g_free); + + /* Check if there are signatures in the superblock */ + g_variant_iter_init (&iter, delta_sign); + while ((item = g_variant_iter_next_value (&iter))) + { + g_autoptr(GVariant) key_v = g_variant_get_child_value (item, 0); + const char *str = g_variant_get_string (key_v, NULL); + if (g_str_has_prefix (str, "ostree.sign.")) + { + ret = TRUE; + if (signatures) + g_ptr_array_add (signatures, g_strdup (str + strlen ("ostree.sign."))); + } + g_variant_unref (item); + } + + if (out_value && ret) + ot_transfer_out_value (out_value, &signatures); + + return ret; +} + /** * ostree_repo_static_delta_execute_offline: * @self: Repo @@ -895,3 +950,76 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, return TRUE; } + +/** + * ostree_repo_static_delta_verify_signature: + * @self: Repo + * @delta_id: delta path + * @sign: Signature engine used to check superblock + * @out_success_message: success message + * @error: Error + * + * Verify static delta file signature. + * + * Returns: TRUE if the signature of static delta file is valid using the + * signature engine provided, FALSE otherwise. + * + * Since: 2020.1 + */ +gboolean +ostree_repo_static_delta_verify_signature (OstreeRepo *self, + const char *delta_id, + OstreeSign *sign, + char **out_success_message, + GError **error) +{ + g_autoptr(GVariantBuilder) desc_sign_builder = NULL; + g_autoptr(GVariant) delta_meta = NULL; + glnx_autofd int delta_fd = -1; + + if (strchr (delta_id, '/')) + { + if (!glnx_openat_rdonly (AT_FDCWD, delta_id, TRUE, &delta_fd, error)) + return FALSE; + } + else + { + g_autofree char *from = NULL; + g_autofree char *to = NULL; + if (!_ostree_parse_delta_name (delta_id, &from, &to, error)) + return FALSE; + + g_autofree char *delta_path = _ostree_get_relative_static_delta_superblock_path (from, to); + if (!glnx_openat_rdonly (self->repo_dir_fd, delta_path, TRUE, &delta_fd, error)) + return FALSE; + } + + if (!_ostree_repo_static_delta_is_signed (self, delta_fd, NULL, error)) + return FALSE; + + g_autoptr(GVariant) delta = NULL; + if (!ot_variant_read_fd (delta_fd, 0, + (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, + TRUE, &delta, error)) + return FALSE; + + /* Check if there are signatures for signature engine */ + const gchar *signature_key = ostree_sign_metadata_key(sign); + GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(sign); + delta_meta = g_variant_get_child_value (delta, 2); + if (delta_meta == NULL) + return glnx_throw (error, "no metadata in static-delta superblock"); + g_autoptr(GVariant) signatures = g_variant_lookup_value (delta_meta, + signature_key, + signature_format); + if (!signatures) + return glnx_throw (error, "no signature for '%s' in static-delta superblock", signature_key); + + /* Get static delta superblock */ + g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); + if (child == NULL) + return glnx_throw (error, "no metadata in static-delta superblock"); + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes(child); + + return ostree_sign_data_verify (sign, signed_data, signatures, out_success_message, error); +} diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index e28af29c..552c1e61 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -32,6 +32,7 @@ #include "ostree-repo-finder.h" #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" +#include "ostree-sign.h" G_BEGIN_DECLS @@ -1074,6 +1075,13 @@ gboolean ostree_repo_static_delta_execute_offline (OstreeRepo GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_static_delta_verify_signature (OstreeRepo *self, + const char *delta_id, + OstreeSign *sign, + char **out_success_message, + GError **error); + _OSTREE_PUBLIC GHashTable *ostree_repo_traverse_new_reachable (void); From 512db0435c506f1d11169dc6f312dd954b06cd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Fri, 29 Nov 2019 12:40:11 +0100 Subject: [PATCH 23/58] bin/static-delta: Add command to verify delta signature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new "static-delta verify" sub-command. This supports multiple keys to verify the static-delta file. Signed-off-by: Frédéric Danis --- bash/ostree | 34 +++++++++++++ man/ostree-static-delta.xml | 64 +++++++++++++++++++++++ src/ostree/ot-builtin-static-delta.c | 76 ++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+) diff --git a/bash/ostree b/bash/ostree index a7389bd7..34a38b20 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1709,6 +1709,40 @@ _ostree_static_delta_show() { return 0 } +_ostree_static_delta_verify() { + local boolean_options=" + $main_boolean_options + " + + local options_with_args=" + --sign-type + --keys-file + --keys-dir + --repo + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --keys-file|--keys-dir|--repo) + __ostree_compreply_dirs_only + return 0 + ;; + $options_with_args_glob ) + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + esac + + return 0 +} + _ostree_static_delta() { local subcommands=" apply-offline diff --git a/man/ostree-static-delta.xml b/man/ostree-static-delta.xml index a4bef237..66fc7590 100644 --- a/man/ostree-static-delta.xml +++ b/man/ostree-static-delta.xml @@ -65,6 +65,9 @@ Boston, MA 02111-1307, USA. ostree static-delta apply-offline PATH + + ostree static-delta verify OPTIONS STATIC-DELTA KEY-ID + @@ -149,6 +152,67 @@ Boston, MA 02111-1307, USA. + + 'Verify' Options + + + + + + + + + + + base64-encoded public key for verifying. + + + + + + + ASCII-string used as public key. + + + + + + + + =ENGINE + + + Use particular signature engine. Currently + available ed25519 and dummy + signature types. + + The default is ed25519. + + + + + + + Read key(s) from file filename. + + + + Valid for ed25519 signature type. + For ed25519 this file must contain base64-encoded + public key(s) per line for verifying. + + + + + + + Redefine the system path, where to search files and subdirectories with + well-known and revoked keys. + + + + + Example diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index d5e93783..4e507e7d 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -43,6 +43,7 @@ static gboolean opt_if_not_exists; static char **opt_key_ids; static char *opt_sign_name; static char *opt_keysfilename; +static char *opt_keysdir; #define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) @@ -51,6 +52,7 @@ BUILTINPROTO(show); BUILTINPROTO(delete); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); +BUILTINPROTO(verify); #undef BUILTINPROTO @@ -70,6 +72,9 @@ static OstreeCommand static_delta_subcommands[] = { { "apply-offline", OSTREE_BUILTIN_FLAG_NONE, ot_static_delta_builtin_apply_offline, "Apply static delta file" }, + { "verify", OSTREE_BUILTIN_FLAG_NONE, + ot_static_delta_builtin_verify, + "Verify static delta signatures" }, { NULL, 0, NULL, NULL } }; @@ -107,6 +112,15 @@ static GOptionEntry list_options[] = { { NULL } }; +static GOptionEntry verify_options[] = { + { "sign-type", 0, 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, +#if defined(HAVE_LIBSODIUM) + { "keys-file", 0, 0, G_OPTION_ARG_STRING, &opt_keysfilename, "Read key(s) from file", "NAME"}, + { "keys-dir", 0, 0, G_OPTION_ARG_STRING, &opt_keysdir, "Redefine system-wide directories with public and revoked keys for verification", "NAME"}, +#endif + { NULL } +}; + static void static_delta_usage (char **argv, gboolean is_error) @@ -439,6 +453,68 @@ ot_static_delta_builtin_apply_offline (int argc, char **argv, OstreeCommandInvoc return TRUE; } +static gboolean +ot_static_delta_builtin_verify (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr (GOptionContext) context = g_option_context_new ("STATIC-DELTA-FILE [KEY-ID...]"); + g_autoptr (OstreeRepo) repo = NULL; + gboolean verified; + char **key_ids; + int n_key_ids; + + if (!ostree_option_context_parse (context, verify_options, &argc, &argv, invocation, &repo, cancellable, error)) + return FALSE; + + if (argc < 3) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "DELTA must be specified"); + return FALSE; + } + + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; + + const char *delta_id = argv[2]; + + g_autoptr (OstreeSign) sign = ostree_sign_get_by_name (opt_sign_name, error); + if (!sign) + { + g_print("Sign-type not supported\n"); + return FALSE; + } + + key_ids = argv + 3; + n_key_ids = argc - 3; + for (int i = 0; i < n_key_ids; i++) + { + g_autoptr (GVariant) pk = g_variant_new_string(key_ids[i]); + if (!ostree_sign_add_pk(sign, pk, error)) + return FALSE; + } + if ((n_key_ids == 0) || opt_keysfilename) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + /* Use custom directory with public and revoked keys instead of system-wide directories */ + if (opt_keysdir) + g_variant_builder_add (builder, "{sv}", "basedir", g_variant_new_string (opt_keysdir)); + /* The last chance for verification source -- system files */ + if (opt_keysfilename) + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_keysfilename)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + return FALSE; + } + + verified = ostree_repo_static_delta_verify_signature (repo, delta_id, sign, NULL, error); + g_print ("Verification %s\n", verified ? "OK" : "fails"); + + return verified; +} + gboolean ostree_builtin_static_delta (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { From 0c48423c267ff80fae1c2918a0e398d5fd292789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Thu, 9 Jul 2020 17:34:08 +0200 Subject: [PATCH 24/58] lib/deltas: Support signed delta in execute_offline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This checks if the static delta file is signed or not to be able to correctly get the superblock to apply. Signed-off-by: Frédéric Danis --- src/libostree/ostree-repo-static-delta-core.c | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 857473db..c71378fa 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -286,6 +286,8 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, GError **error) { g_autofree char *basename = NULL; + g_autoptr(GVariant) delta = NULL; + g_autoptr(GVariant) meta = NULL; const char *dir_or_file_path = gs_file_get_path_cached (dir_or_file); @@ -311,10 +313,24 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, if (meta_fd < 0) return glnx_throw_errno_prefix (error, "openat(%s)", basename); - g_autoptr(GVariant) meta = NULL; - if (!ot_variant_read_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), - FALSE, &meta, error)) - return FALSE; + gboolean is_signed = _ostree_repo_static_delta_is_signed (self, meta_fd, NULL, NULL); + if (is_signed) + { + if (!ot_variant_read_fd (meta_fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, + TRUE, &delta, error)) + return FALSE; + + g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); + g_autoptr(GBytes) bytes = g_variant_get_data_as_bytes (child); + meta = g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, + bytes, FALSE); + } + else + { + if (!ot_variant_read_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), + FALSE, &meta, error)) + return FALSE; + } /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ From bf0c09ffe19df69cb836aff7421db717e5f0ce48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Thu, 9 Jul 2020 17:35:00 +0200 Subject: [PATCH 25/58] lib/deltas: Support signed delta in dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This checks if the static delta file is signed or not to be able to correctly get the superblock to dump. Signed-off-by: Frédéric Danis --- src/libostree/ostree-repo-static-delta-core.c | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index c71378fa..8d0f4e82 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -819,6 +819,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, GError **error) { glnx_autofd int superblock_fd = -1; + g_autoptr(GVariant) delta = NULL; + g_autoptr(GVariant) delta_superblock = NULL; if (strchr (delta_id, '/')) { @@ -837,13 +839,28 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, return FALSE; } - g_autoptr(GVariant) delta_superblock = NULL; - if (!ot_variant_read_fd (superblock_fd, 0, - (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, - TRUE, &delta_superblock, error)) - return FALSE; + gboolean is_signed = _ostree_repo_static_delta_is_signed(self, superblock_fd, NULL, NULL); + if (is_signed) + { + if (!ot_variant_read_fd (superblock_fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, + TRUE, &delta, error)) + return FALSE; + + g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); + g_autoptr(GBytes) bytes = g_variant_get_data_as_bytes(child); + delta_superblock = g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, + bytes, FALSE); + } + else + { + if (!ot_variant_read_fd (superblock_fd, 0, + (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, + TRUE, &delta_superblock, error)) + return FALSE; + } g_print ("Delta: %s\n", delta_id); + g_print ("Signed: %s\n", is_signed ? "yes" : "no"); g_autoptr(GVariant) from_commit_v = NULL; g_variant_get_child (delta_superblock, 2, "@ay", &from_commit_v); g_autofree char *from_commit = NULL; From c98a993c996713e16af8e36cbbde63f6000d1341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Fri, 29 Nov 2019 16:17:17 +0100 Subject: [PATCH 26/58] tests/delta: new tests for signed deltas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests to generate signed deltas and verify them using 'dummy' signature engine. Signed-off-by: Frédéric Danis --- Makefile-tests.am | 1 + tests/test-delta-sign.sh | 131 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100755 tests/test-delta-sign.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index a4179377..e463178b 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -114,6 +114,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-reset-nonlinear.sh \ tests/test-oldstyle-partial.sh \ tests/test-delta.sh \ + tests/test-delta-sign.sh \ tests/test-xattrs.sh \ tests/test-auto-summary.sh \ tests/test-prune.sh \ diff --git a/tests/test-delta-sign.sh b/tests/test-delta-sign.sh new file mode 100755 index 00000000..b9854ce7 --- /dev/null +++ b/tests/test-delta-sign.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# +# Copyright (C) 2011,2013 Colin Walters +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +skip_without_user_xattrs + +bindatafiles="bash true ostree" + +echo '1..3' + +# This is explicitly opt in for testing +export OSTREE_DUMMY_SIGN_ENABLED=1 + +mkdir repo +ostree_repo_init repo --mode=archive + +mkdir files +for bin in ${bindatafiles}; do + cp $(which ${bin}) files +done + +${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files + +function permuteFile() { + permutation=$(($1 % 2)) + output=$2 + case $permutation in + 0) dd if=/dev/zero count=40 bs=1 >> $output;; + 1) echo aheader | cat - $output >> $output.new && mv $output.new $output;; + esac +} + +function permuteDirectory() { + permutation=$1 + dir=$2 + for x in ${dir}/*; do + for z in $(seq ${permutation}); do + permuteFile ${z} ${x} + done + done +} + +get_assert_one_direntry_matching() { + local path=$1 + local r=$2 + local child="" + local bn + for p in ${path}/*; do + bn=$(basename $p) + if ! echo ${bn} | grep -q "$r"; then + continue + fi + if test -z "${child}"; then + child=${bn} + else + assert_not_reached "Expected only one child matching ${r} in ${path}"; + fi + done + if test -z "${child}"; then + assert_not_reached "Failed to find child matching ${r}" + fi + echo ${child} +} + +origrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse test) + +permuteDirectory 1 files +${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files + +newrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse test) + +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} dummysign > show-not-signed.txt 2>&1 && exit 1 +assert_file_has_content show-not-signed.txt "Verification fails" +assert_file_has_content show-not-signed.txt "no signatures in static-delta" + +deltaprefix=$(get_assert_one_direntry_matching repo/deltas '.') +deltadir=$(get_assert_one_direntry_matching repo/deltas/${deltaprefix} '-') + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} dummysign > show-inline-not-signed.txt 2>&1 && exit 1 +assert_file_has_content show-not-signed.txt "Verification fails" +assert_file_has_content show-not-signed.txt "no signatures in static-delta" + +echo 'ok verify ok with unsigned deltas' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=dummy --sign=dummysign +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} dummysign > show-dummy-signed.txt +assert_file_has_content show-dummy-signed.txt "Verification OK" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=dummy --sign=dummysign +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} dummysign > show-dummy-inline-signed.txt +assert_file_has_content show-dummy-inline-signed.txt "Verification OK" + +echo 'ok verified with dummy' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=dummy --sign=dummysign +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} badsign > show-dummy-bad-signed.txt && exit 1 +assert_file_has_content show-dummy-bad-signed.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=dummy --sign=dummysign +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev}-${newrev} badsign > show-dummy-bad-inline-signed.txt && exit 1 +assert_file_has_content show-dummy-bad-inline-signed.txt "Verification fails" + +echo 'ok verification failed with dummy and bad key' From 96bcc25632fa0b1b4de637845bf43535f4eb3628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Thu, 23 Apr 2020 15:24:53 +0200 Subject: [PATCH 27/58] tests/libtest.sh: Add skip_without_sign_ed25519() function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Frédéric Danis --- tests/libtest.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/libtest.sh b/tests/libtest.sh index ca457fa2..7c66a5c6 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -700,6 +700,12 @@ has_sign_ed25519 () { return ${ret} } +skip_without_sign_ed25519() { + if ! has_sign_ed25519; then + skip "no ed25519 support compiled in" + fi +} + # Keys for ed25519 signing tests ED25519PUBLIC= ED25519SEED= From 869dbc037e92fbe2d859e8fda97bc4a287160770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Tue, 3 Dec 2019 11:15:51 +0100 Subject: [PATCH 28/58] tests/delta: new tests for 'ed25519' signed deltas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests to generate signed deltas and verify them using 'ed25519' signature engine. Signed-off-by: Frédéric Danis --- Makefile-tests.am | 1 + tests/test-delta-ed25519.sh | 283 ++++++++++++++++++++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100755 tests/test-delta-ed25519.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index e463178b..3e4f17a5 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -115,6 +115,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-oldstyle-partial.sh \ tests/test-delta.sh \ tests/test-delta-sign.sh \ + tests/test-delta-ed25519.sh \ tests/test-xattrs.sh \ tests/test-auto-summary.sh \ tests/test-prune.sh \ diff --git a/tests/test-delta-ed25519.sh b/tests/test-delta-ed25519.sh new file mode 100755 index 00000000..e50b9763 --- /dev/null +++ b/tests/test-delta-ed25519.sh @@ -0,0 +1,283 @@ +#!/bin/bash +# +# Copyright (C) 2011,2013 Colin Walters +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +skip_without_user_xattrs + +skip_without_sign_ed25519 + +bindatafiles="bash true ostree" + +echo '1..9' + +mkdir repo +ostree_repo_init repo --mode=archive + +mkdir files +for bin in ${bindatafiles}; do + cp $(which ${bin}) files +done + +${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files + +function permuteFile() { + permutation=$(($1 % 2)) + output=$2 + case $permutation in + 0) dd if=/dev/zero count=40 bs=1 >> $output;; + 1) echo aheader | cat - $output >> $output.new && mv $output.new $output;; + esac +} + +function permuteDirectory() { + permutation=$1 + dir=$2 + for x in ${dir}/*; do + for z in $(seq ${permutation}); do + permuteFile ${z} ${x} + done + done +} + +get_assert_one_direntry_matching() { + local path=$1 + local r=$2 + local child="" + local bn + for p in ${path}/*; do + bn=$(basename $p) + if ! echo ${bn} | grep -q "$r"; then + continue + fi + if test -z "${child}"; then + child=${bn} + else + assert_not_reached "Expected only one child matching ${r} in ${path}"; + fi + done + if test -z "${child}"; then + assert_not_reached "Failed to find child matching ${r}" + fi + echo ${child} +} + +origrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse test) + +permuteDirectory 1 files +${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files + +newrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse test) + +# Test ostree sign with 'ed25519' module +gen_ed25519_keys +PUBLIC=${ED25519PUBLIC} +SEED=${ED25519SEED} +SECRET=${ED25519SECRET} +WRONG_PUBLIC="$(gen_ed25519_random_public)" + +SECRETKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" +echo ${SECRET} > ${SECRETKEYS} + +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-key-signed-1.txt +assert_file_has_content show-ed25519-key-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" "${WRONG_PUBLIC}" > show-ed25519-key-signed-2.txt +assert_file_has_content show-ed25519-key-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" "${PUBLIC}" > show-ed25519-key-signed-3.txt +assert_file_has_content show-ed25519-key-signed-3.txt "Verification OK" + +deltaprefix=$(get_assert_one_direntry_matching repo/deltas '.') +deltadir=$(get_assert_one_direntry_matching repo/deltas/${deltaprefix} '-') + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-key-inline-signed-1.txt +assert_file_has_content show-ed25519-key-inline-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" "${WRONG_PUBLIC}" > show-ed25519-key-inline-signed-2.txt +assert_file_has_content show-ed25519-key-inline-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" "${PUBLIC}" > show-ed25519-key-inline-signed-3.txt +assert_file_has_content show-ed25519-key-inline-signed-3.txt "Verification OK" + +echo 'ok verified with ed25519 (sign - key)' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-keyfile-signed-1.txt +assert_file_has_content show-ed25519-keyfile-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" "${WRONG_PUBLIC}" > show-ed25519-keyfile-signed-2.txt +assert_file_has_content show-ed25519-keyfile-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" "${PUBLIC}" > show-ed25519-keyfile-signed-3.txt +assert_file_has_content show-ed25519-keyfile-signed-3.txt "Verification OK" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-keyfile-inline-signed-1.txt +assert_file_has_content show-ed25519-keyfile-inline-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" "${WRONG_PUBLIC}" > show-ed25519-keyfile-inline-signed-2.txt +assert_file_has_content show-ed25519-keyfile-inline-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" "${PUBLIC}" > show-ed25519-keyfile-inline-signed-3.txt +assert_file_has_content show-ed25519-keyfile-inline-signed-3.txt "Verification OK" + +echo 'ok verified with ed25519 (keyfile - key)' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-key-bad-signed.txt && exit 1 +assert_file_has_content show-ed25519-key-bad-signed.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-key-bad-inline-signed.txt && exit 1 +assert_file_has_content show-ed25519-key-bad-inline-signed.txt "Verification fails" + +echo 'ok Verification fails with ed25519 (sign - bad key)' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-keyfile-bad-signed.txt && exit 1 +assert_file_has_content show-ed25519-keyfile-bad-signed.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-keyfile-bad-inline-signed.txt && exit 1 +assert_file_has_content show-ed25519-keyfile-bad-inline-signed.txt "Verification fails" + +echo 'ok Verification fails with ed25519 (keyfile - bad key)' + +# Prepare files with public ed25519 signatures +PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" +for((i=0;i<100;i++)); do + # Generate a list with some public signatures + gen_ed25519_random_public +done > ${PUBKEYS} + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-bad-signed-1.txt && exit 1 +assert_file_has_content show-ed25519-file-bad-signed-1.txt "Verification fails" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-bad-signed-2.txt && exit 1 +assert_file_has_content show-ed25519-file-bad-signed-2.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-inline-bad-signed-1.txt && exit 1 +assert_file_has_content show-ed25519-file-inline-bad-signed-1.txt "Verification fails" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-inline-bad-signed-2.txt && exit 1 +assert_file_has_content show-ed25519-file-inline-bad-signed-2.txt "Verification fails" + +echo 'ok Verification fails with ed25519 (sign - bad keys)' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-bad-signed-3.txt && exit 1 +assert_file_has_content show-ed25519-file-bad-signed-3.txt "Verification fails" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-bad-signed-4.txt && exit 1 +assert_file_has_content show-ed25519-file-bad-signed-4.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-inline-bad-signed-3.txt && exit 1 +assert_file_has_content show-ed25519-file-inline-bad-signed-3.txt "Verification fails" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-inline-bad-signed-4.txt && exit 1 +assert_file_has_content show-ed25519-file-inline-bad-signed-4.txt "Verification fails" + +echo 'ok Verification fails with ed25519 (keyfile - bad keys)' + +# Add correct key into the list +echo ${PUBLIC} >> ${PUBKEYS} + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-signed-1.txt +assert_file_has_content show-ed25519-file-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-signed-2.txt +assert_file_has_content show-ed25519-file-signed-2.txt "Verification OK" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --sign=${SECRET} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-inline-signed-1.txt +assert_file_has_content show-ed25519-file-inline-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-inline-signed-2.txt +assert_file_has_content show-ed25519-file-inline-signed-2.txt "Verification OK" + +echo 'ok verified with ed25519 (sign - file)' + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-signed-3.txt +assert_file_has_content show-ed25519-file-signed-3.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-signed-4.txt +assert_file_has_content show-ed25519-file-signed-4.txt "Verification OK" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-file-inline-signed-3.txt +assert_file_has_content show-ed25519-file-inline-signed-3.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-file-inline-signed-4.txt +assert_file_has_content show-ed25519-file-inline-signed-4.txt "Verification OK" + +echo 'ok verified with ed25519 (keyfile - file)' + +# Test ostree sign with multiple 'ed25519' keys +gen_ed25519_keys +PUBLIC2=${ED25519PUBLIC} +SEED2=${ED25519SEED} +SECRET2=${ED25519SECRET} + +echo ${SECRET2} >> ${SECRETKEYS} +echo ${PUBLIC2} >> ${PUBKEYS} + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-multiplekeys-signed-1.txt +assert_file_has_content show-ed25519-multiplekeys-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC2}" > show-ed25519-multiplekeys-signed-2.txt +assert_file_has_content show-ed25519-multiplekeys-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-multiplekeys-bad-signed.txt && exit 1 +assert_file_has_content show-ed25519-multiplekeys-bad-signed.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-multiplekeys-signed-3.txt +assert_file_has_content show-ed25519-multiplekeys-signed-3.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-multiplekeys-signed-4.txt +assert_file_has_content show-ed25519-multiplekeys-signed-4.txt "Verification OK" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC}" > show-ed25519-multiplekeys-inline-signed-1.txt +assert_file_has_content show-ed25519-multiplekeys-inline-signed-1.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${PUBLIC2}" > show-ed25519-multiplekeys-inline-signed-2.txt +assert_file_has_content show-ed25519-multiplekeys-inline-signed-2.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} "${WRONG_PUBLIC}" > show-ed25519-multiplekeys-bad-inline-signed.txt && exit 1 +assert_file_has_content show-ed25519-multiplekeys-bad-inline-signed.txt "Verification fails" + +rm -rf repo/deltas/${deltaprefix}/${deltadir}/* +${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --inline --sign-type=ed25519 --keys-file=${SECRETKEYS} +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} > show-ed25519-multiplekeys-inline-signed-3.txt +assert_file_has_content show-ed25519-multiplekeys-inline-signed-3.txt "Verification OK" +${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origrev}-${newrev} --keys-file=${PUBKEYS} "${WRONG_PUBLIC}" > show-ed25519-multiplekeys-inline-signed-4.txt +assert_file_has_content show-ed25519-multiplekeys-inline-signed-4.txt "Verification OK" + +echo 'ok verified with ed25519 (multiple keys)' From fb1faf17d6f9cefd349c46f48f7a28f269f07576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Tue, 25 Aug 2020 09:26:09 +0200 Subject: [PATCH 29/58] lib/deltas: Check signed delta in execute_offline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new function `ostree_repo_static_delta_execute_offline_with_signature` which takes a signature engine to verify the delta before applying it. The `ostree_repo_static_delta_execute_offline` is just a wrapper to this new function, passing a NULL signature engine. When this function is called without signature engine, but with a sign delta, it will only fails if `sign-verify-deltas` is set to true in repo core options. This commits move signature existence check and delta signature verification to share common parts between existing APIs and the new function. Signed-off-by: Frédéric Danis --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-static-delta-core.c | 132 +++++++++++++----- src/libostree/ostree-repo.h | 8 ++ 4 files changed, 108 insertions(+), 34 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 8cafeb1f..66b42233 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -412,6 +412,7 @@ ostree_repo_list_commit_objects_starting_with ostree_repo_list_static_delta_names OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate +ostree_repo_static_delta_execute_offline_with_signature ostree_repo_static_delta_execute_offline ostree_repo_static_delta_verify_signature ostree_repo_traverse_new_reachable diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 9a847b92..a15654c1 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -22,6 +22,7 @@ global: /* Add symbols here, and uncomment the bits in * Makefile-libostree.am to enable this too. */ + ostree_repo_static_delta_execute_offline_with_signature; ostree_repo_static_delta_verify_signature; } LIBOSTREE_2020.4; diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 8d0f4e82..b16764ae 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -265,25 +265,68 @@ _ostree_repo_static_delta_is_signed (OstreeRepo *self, return ret; } +static gboolean +_ostree_repo_static_delta_verify_signature (OstreeRepo *self, + int fd, + OstreeSign *sign, + char **out_success_message, + GError **error) +{ + g_autoptr(GVariantBuilder) desc_sign_builder = NULL; + g_autoptr(GVariant) delta_meta = NULL; + g_autoptr(GVariant) delta = NULL; + + if (!ot_variant_read_fd (fd, 0, + (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, + TRUE, &delta, error)) + return FALSE; + + /* Check if there are signatures for signature engine */ + const gchar *signature_key = ostree_sign_metadata_key(sign); + GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(sign); + delta_meta = g_variant_get_child_value (delta, 2); + if (delta_meta == NULL) + return glnx_throw (error, "no metadata in static-delta superblock"); + g_autoptr(GVariant) signatures = g_variant_lookup_value (delta_meta, + signature_key, + signature_format); + if (!signatures) + return glnx_throw (error, "no signature for '%s' in static-delta superblock", signature_key); + + /* Get static delta superblock */ + g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); + if (child == NULL) + return glnx_throw (error, "no metadata in static-delta superblock"); + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes(child); + + return ostree_sign_data_verify (sign, signed_data, signatures, out_success_message, error); +} + /** - * ostree_repo_static_delta_execute_offline: + * ostree_repo_static_delta_execute_offline_with_signature: * @self: Repo * @dir_or_file: Path to a directory containing static delta data, or directly to the superblock + * @sign: Signature engine used to check superblock * @skip_validation: If %TRUE, assume data integrity * @cancellable: Cancellable * @error: Error * * Given a directory representing an already-downloaded static delta - * on disk, apply it, generating a new commit. The directory must be - * named with the form "FROM-TO", where both are checksums, and it - * must contain a file named "superblock", along with at least one part. + * on disk, apply it, generating a new commit. + * If sign is passed, the static delta signature is verified. + * If sign-verify-deltas configuration option is set and static delta is signed, + * signature verification will be mandatory before apply the static delta. + * The directory must be named with the form "FROM-TO", where both are + * checksums, and it must contain a file named "superblock", along with at least + * one part. */ gboolean -ostree_repo_static_delta_execute_offline (OstreeRepo *self, - GFile *dir_or_file, - gboolean skip_validation, - GCancellable *cancellable, - GError **error) +ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, + GFile *dir_or_file, + OstreeSign *sign, + gboolean skip_validation, + GCancellable *cancellable, + GError **error) { g_autofree char *basename = NULL; g_autoptr(GVariant) delta = NULL; @@ -316,6 +359,25 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, gboolean is_signed = _ostree_repo_static_delta_is_signed (self, meta_fd, NULL, NULL); if (is_signed) { + gboolean verify_deltas; + gboolean verified; + + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "sign-verify-deltas", + FALSE, &verify_deltas, error)) + return FALSE; + + if (verify_deltas && !sign) + return glnx_throw (error, "Key is mandatory to check delta signature"); + + if (sign) + { + verified = _ostree_repo_static_delta_verify_signature (self, meta_fd, sign, NULL, error); + if (*error) + return FALSE; + if (!verified) + return glnx_throw (error, "Delta signature verification failed"); + } + if (!ot_variant_read_fd (meta_fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, TRUE, &delta, error)) return FALSE; @@ -479,6 +541,32 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, return TRUE; } +/** + * ostree_repo_static_delta_execute_offline: + * @self: Repo + * @dir_or_file: Path to a directory containing static delta data, or directly to the superblock + * @skip_validation: If %TRUE, assume data integrity + * @cancellable: Cancellable + * @error: Error + * + * Given a directory representing an already-downloaded static delta + * on disk, apply it, generating a new commit. The directory must be + * named with the form "FROM-TO", where both are checksums, and it + * must contain a file named "superblock", along with at least one part. + */ +gboolean +ostree_repo_static_delta_execute_offline (OstreeRepo *self, + GFile *dir_or_file, + gboolean skip_validation, + GCancellable *cancellable, + GError **error) +{ + return ostree_repo_static_delta_execute_offline_with_signature(self, dir_or_file, NULL, + skip_validation, + cancellable, + error); +} + gboolean _ostree_static_delta_part_open (GInputStream *part_in, GBytes *inline_part_bytes, @@ -1030,29 +1118,5 @@ ostree_repo_static_delta_verify_signature (OstreeRepo *self, if (!_ostree_repo_static_delta_is_signed (self, delta_fd, NULL, error)) return FALSE; - g_autoptr(GVariant) delta = NULL; - if (!ot_variant_read_fd (delta_fd, 0, - (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, - TRUE, &delta, error)) - return FALSE; - - /* Check if there are signatures for signature engine */ - const gchar *signature_key = ostree_sign_metadata_key(sign); - GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(sign); - delta_meta = g_variant_get_child_value (delta, 2); - if (delta_meta == NULL) - return glnx_throw (error, "no metadata in static-delta superblock"); - g_autoptr(GVariant) signatures = g_variant_lookup_value (delta_meta, - signature_key, - signature_format); - if (!signatures) - return glnx_throw (error, "no signature for '%s' in static-delta superblock", signature_key); - - /* Get static delta superblock */ - g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); - if (child == NULL) - return glnx_throw (error, "no metadata in static-delta superblock"); - g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes(child); - - return ostree_sign_data_verify (sign, signed_data, signatures, out_success_message, error); + return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error); } diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 552c1e61..d52fc9c3 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1068,6 +1068,14 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, + GFile *dir_or_file, + OstreeSign *sign, + gboolean skip_validation, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_static_delta_execute_offline (OstreeRepo *self, GFile *dir_or_file, From 2e97f5659ffdc52bcdd5906d2e98fb1013b3c2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Fri, 21 Aug 2020 17:22:40 +0200 Subject: [PATCH 30/58] bin/static-delta: Add signature parameters to apply-offline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to check the delta signature before applying it. Signed-off-by: Frédéric Danis --- bash/ostree | 3 ++ man/ostree-static-delta.xml | 61 +++++++++++++++++++++++++++- src/ostree/ot-builtin-static-delta.c | 56 ++++++++++++++++++++++++- 3 files changed, 118 insertions(+), 2 deletions(-) diff --git a/bash/ostree b/bash/ostree index 34a38b20..d00695ef 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1532,6 +1532,9 @@ _ostree_static_delta_apply_offline() { " local options_with_args=" + --sign-type + --keys-file + --keys-dir --repo " diff --git a/man/ostree-static-delta.xml b/man/ostree-static-delta.xml index 66fc7590..440ada41 100644 --- a/man/ostree-static-delta.xml +++ b/man/ostree-static-delta.xml @@ -63,7 +63,7 @@ Boston, MA 02111-1307, USA. ostree static-delta generate --to=REV OPTIONS - ostree static-delta apply-offline PATH + ostree static-delta apply-offline OPTIONS PATH KEY-ID ostree static-delta verify OPTIONS STATIC-DELTA KEY-ID @@ -152,6 +152,65 @@ Boston, MA 02111-1307, USA. + + 'Apply-offline' Options + + + + + + + + + + + base64-encoded public key for verifying. + + + + + + + ASCII-string used as public key. + + + + + + + + =ENGINE + + + Use particular signature engine. Currently + available ed25519 and dummy + signature types. + + + + + + + Read key(s) from file filename. + + + + Valid for ed25519 signature type. + For ed25519 this file must contain base64-encoded + public key(s) per line for verifying. + + + + + + + Redefine the system path, where to search files and subdirectories with + well-known and revoked keys. + + + + + 'Verify' Options diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 4e507e7d..3e0af5bd 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -105,6 +105,11 @@ static GOptionEntry generate_options[] = { }; static GOptionEntry apply_offline_options[] = { + { "sign-type", 0, 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, +#if defined(HAVE_LIBSODIUM) + { "keys-file", 0, 0, G_OPTION_ARG_STRING, &opt_keysfilename, "Read key(s) from file", "NAME"}, + { "keys-dir", 0, 0, G_OPTION_ARG_STRING, &opt_keysdir, "Redefine system-wide directories with public and revoked keys for verification", "NAME"}, +#endif { NULL } }; @@ -423,6 +428,9 @@ ot_static_delta_builtin_apply_offline (int argc, char **argv, OstreeCommandInvoc { g_autoptr(GOptionContext) context = NULL; g_autoptr(OstreeRepo) repo = NULL; + g_autoptr (OstreeSign) sign = NULL; + char **key_ids; + int n_key_ids; context = g_option_context_new (""); if (!ostree_option_context_parse (context, apply_offline_options, &argc, &argv, invocation, &repo, cancellable, error)) @@ -438,13 +446,59 @@ ot_static_delta_builtin_apply_offline (int argc, char **argv, OstreeCommandInvoc return FALSE; } +#if defined(HAVE_LIBSODIUM) + /* Initialize crypto system */ + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; +#endif + + if (opt_sign_name) + { + sign = ostree_sign_get_by_name (opt_sign_name, error); + if (!sign) + return glnx_throw (error, "Signing type %s is not supported", opt_sign_name); + + key_ids = argv + 3; + n_key_ids = argc - 3; + for (int i = 0; i < n_key_ids; i++) + { + g_autoptr (GVariant) pk = g_variant_new_string(key_ids[i]); + if (!ostree_sign_add_pk(sign, pk, error)) + return FALSE; + } + if ((n_key_ids == 0) || opt_keysfilename) + { + g_autoptr (GVariantBuilder) builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_autoptr (GVariant) options = NULL; + + /* Use custom directory with public and revoked keys instead of system-wide directories */ + if (opt_keysdir) + g_variant_builder_add (builder, "{sv}", "basedir", g_variant_new_string (opt_keysdir)); + /* The last chance for verification source -- system files */ + if (opt_keysfilename) + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_keysfilename)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + { + /* If it fails to load system default public keys, consider there no signature engine */ + if (!opt_keysdir && !opt_keysfilename) + { + g_clear_error(error); + g_clear_object(&sign); + } + else + return FALSE; + } + } + } + const char *patharg = argv[2]; g_autoptr(GFile) path = g_file_new_for_path (patharg); if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) return FALSE; - if (!ostree_repo_static_delta_execute_offline (repo, path, FALSE, cancellable, error)) + if (!ostree_repo_static_delta_execute_offline_with_signature (repo, path, sign, FALSE, cancellable, error)) return FALSE; if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error)) From ecbfe08ec75497767b76a962319f7bff6449da0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Fri, 21 Aug 2020 17:24:49 +0200 Subject: [PATCH 31/58] tests/delta: Add new tests for applying signed deltas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new test to apply offline signed deltas. Signed-off-by: Frédéric Danis --- tests/test-delta-ed25519.sh | 41 ++++++++++++++++++++++++++++++++- tests/test-delta-sign.sh | 45 ++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/tests/test-delta-ed25519.sh b/tests/test-delta-ed25519.sh index e50b9763..ef732cf9 100755 --- a/tests/test-delta-ed25519.sh +++ b/tests/test-delta-ed25519.sh @@ -29,7 +29,7 @@ skip_without_sign_ed25519 bindatafiles="bash true ostree" -echo '1..9' +echo '1..12' mkdir repo ostree_repo_init repo --mode=archive @@ -281,3 +281,42 @@ ${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=ed25519 ${origr assert_file_has_content show-ed25519-multiplekeys-inline-signed-4.txt "Verification OK" echo 'ok verified with ed25519 (multiple keys)' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --sign-type=ed25519 --keys-file=${PUBKEYS} repo/deltas/${deltaprefix}/${deltadir} +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null + +echo 'ok apply offline with ed25519 (keyfile)' + +mkdir -p ${test_tmpdir}/{trusted,revoked}.ed25519.d + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +echo ${PUBLIC} > ${test_tmpdir}/trusted.ed25519.d/correct +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --keys-dir=${test_tmpdir} repo/deltas/${deltaprefix}/${deltadir} +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null + +echo 'ok apply offline with ed25519 (keydir)' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +echo ${PUBLIC} > ${test_tmpdir}/revoked.ed25519.d/correct +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +if ${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --keys-dir=${test_tmpdir} repo/deltas/${deltaprefix}/${deltadir}; then + exit 1 +fi + +rm -rf ${test_tmpdir}/{trusted,revoked}.ed25519.d + +echo 'ok apply offline with ed25519 revoking key mechanism (keydir)' diff --git a/tests/test-delta-sign.sh b/tests/test-delta-sign.sh index b9854ce7..86f12f96 100755 --- a/tests/test-delta-sign.sh +++ b/tests/test-delta-sign.sh @@ -27,7 +27,7 @@ skip_without_user_xattrs bindatafiles="bash true ostree" -echo '1..3' +echo '1..7' # This is explicitly opt in for testing export OSTREE_DUMMY_SIGN_ENABLED=1 @@ -129,3 +129,46 @@ ${CMD_PREFIX} ostree --repo=repo static-delta verify --sign-type=dummy ${origrev assert_file_has_content show-dummy-bad-inline-signed.txt "Verification fails" echo 'ok verification failed with dummy and bad key' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline repo/deltas/${deltaprefix}/${deltadir} +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null + +echo 'ok apply offline with no signature verification and no key' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +${CMD_PREFIX} ostree --repo=repo2 config set core.sign-verify-deltas true +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline repo/deltas/${deltaprefix}/${deltadir} 2> apply-offline-verification-no-key.txt && exit 1 +assert_file_has_content apply-offline-verification-no-key.txt "Key is mandatory to check delta signature" + +echo 'ok apply offline failed with signature verification forced and no key' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --sign-type=dummy repo/deltas/${deltaprefix}/${deltadir} dummysign +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null + +echo 'ok apply offline with dummy' + +rm -rf repo2 +ostree_repo_init repo2 --mode=bare-user + +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --sign-type=dummy repo/deltas/${deltaprefix}/${deltadir} badsign 2> apply-offline-bad-key.txt && exit 1 +assert_file_has_content apply-offline-bad-key.txt "signature: dummy: incorrect signature" + +echo 'ok apply offline failed with dummy and bad key' From 3441a48c5835e9470a77032ec150cde87aabe242 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 16 Sep 2020 13:23:04 +0000 Subject: [PATCH 32/58] checkout: Ensure copies of unreadable usermode checkouts are readable The extreme special case of "zero mode" files like `/etc/shadow` comes up again. What we want is for "user mode" checkouts to override it to make the file readable; otherwise when operating as non-root without `CAP_DAC_OVERRIDE` it becomes very difficult to work with. Previously, we were hardlinking these files, but then it intersects with *another* special case around zero sized files, which is *also* true for `/etc/shadow`. Trying to avoid hardlinking there unveiled this bug - when we go to do a copy checkout, we need to override the mode. --- src/libostree/ostree-repo-checkout.c | 20 ++++++++++++++++---- tests/test-basic-user.sh | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index dc36370f..10535a35 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -613,9 +613,12 @@ checkout_one_file_at (OstreeRepo *repo, } const gboolean is_symlink = (g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK); + const guint32 source_mode = g_file_info_get_attribute_uint32 (source_info, "unix::mode"); + const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0); const gboolean is_whiteout = (!is_symlink && options->process_whiteouts && g_str_has_prefix (destination_name, WHITEOUT_PREFIX)); const gboolean is_reg_zerosized = (!is_symlink && g_file_info_get_size (source_info) == 0); + const gboolean override_user_unreadable = (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER && is_unreadable); /* First, see if it's a Docker whiteout, * https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go @@ -634,7 +637,7 @@ checkout_one_file_at (OstreeRepo *repo, need_copy = FALSE; } - else if (options->force_copy_zerosized && is_reg_zerosized) + else if ((options->force_copy_zerosized && is_reg_zerosized) || override_user_unreadable) { need_copy = TRUE; } @@ -735,7 +738,7 @@ checkout_one_file_at (OstreeRepo *repo, if (can_cache && !is_whiteout && !is_symlink - && !is_reg_zerosized + && !(is_reg_zerosized || override_user_unreadable) && need_copy && repo->mode == OSTREE_REPO_MODE_ARCHIVE && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) @@ -799,12 +802,21 @@ checkout_one_file_at (OstreeRepo *repo, * succeeded at hardlinking above. */ if (options->no_copy_fallback) - g_assert (is_bare_user_symlink || is_reg_zerosized); + g_assert (is_bare_user_symlink || is_reg_zerosized || override_user_unreadable); if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) return FALSE; - if (!create_file_copy_from_input_at (repo, options, state, checksum, source_info, xattrs, input, + GFileInfo *copy_source_info = source_info; + g_autoptr(GFileInfo) modified_info = NULL; + if (override_user_unreadable) + { + modified_info = g_file_info_dup (source_info); + g_file_info_set_attribute_uint32 (modified_info, "unix::mode", (source_mode | S_IRUSR)); + copy_source_info = modified_info; + } + + if (!create_file_copy_from_input_at (repo, options, state, checksum, copy_source_info, xattrs, input, destination_dfd, destination_name, cancellable, error)) return glnx_prefix_error (error, "Copy checkout of %s to %s", checksum, destination_name); diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index e56f828a..fa17beee 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -75,6 +75,8 @@ $OSTREE fsck rm test2-checkout -rf $OSTREE checkout -U -H test2-unreadable test2-checkout assert_file_has_mode test2-checkout/unreadable 400 +# Should not be hardlinked +assert_streq $(stat -c "%h" test2-checkout/unreadable) 1 echo "ok bare-user handled unreadable file" cd ${test_tmpdir} From aa2a2783ea71d956db41f0f874d596752e47449f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 24 Sep 2020 19:28:31 +0000 Subject: [PATCH 33/58] deploy: Remove deployment bootcsum assertion When support for devicetree was added, it created a problem because old and new ostree versions would compute different checksums for the "boot data". The scenario here is: - Have system with ostree < 2020.4 - Reboot into system with ostree 2020.5 - Try to perform an operation that would retain that previous booted deployment (common) Currently ostree iterates over all the deployments that will be retained and calls `install_deployment_kernel()`, even for the booted one (which is a bit silly), but just to verify that all boot data for the targeted deployments are installed. This then re-computes the checksum and we'd trip this assertion. In practice though, we don't strictly require them to match; the only thing that will happen if they don't is that we'll end up with another copy of the kernel/initramfs - and that only temporarily until the previous deployment gets GC'd. Longer term, I think what we really want to do anyways is probably closer to like a little ostree repo for `/boot` so that we can e.g. still hardlink kernels there even if the initramfs changes, or hardlink both kernel/initramfs if just the devicetree changes, etc. Closes: https://github.com/ostreedev/ostree/issues/2154 --- Makefile-tests.am | 1 + src/libostree/ostree-sysroot-deploy.c | 67 ++++++++++++++------------ src/libostree/ostree-sysroot-private.h | 1 + src/libostree/ostree-sysroot.c | 1 + tests/test-osupdate-dtb.sh | 62 ++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 31 deletions(-) create mode 100755 tests/test-osupdate-dtb.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index a4179377..eb110dad 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -105,6 +105,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-admin-deploy-nomerge.sh \ tests/test-admin-deploy-none.sh \ tests/test-admin-deploy-bootid-gc.sh \ + tests/test-osupdate-dtb.sh \ tests/test-admin-instutil-set-kargs.sh \ tests/test-admin-upgrade-not-backwards.sh \ tests/test-admin-pull-deploy-commit.sh \ diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 3f337a50..d97d96a8 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1028,7 +1028,8 @@ _ostree_kernel_layout_new (void) /* See get_kernel_from_tree() below */ static gboolean -get_kernel_from_tree_usrlib_modules (int deployment_dfd, +get_kernel_from_tree_usrlib_modules (OstreeSysroot *sysroot, + int deployment_dfd, OstreeKernelLayout **out_layout, GCancellable *cancellable, GError **error) @@ -1137,37 +1138,41 @@ get_kernel_from_tree_usrlib_modules (int deployment_dfd, g_clear_object (&in); glnx_close_fd (&fd); - /* Check for /usr/lib/modules/$kver/devicetree first, if it does not - * exist check for /usr/lib/modules/$kver/dtb/ directory. - */ - if (!ot_openat_ignore_enoent (ret_layout->boot_dfd, "devicetree", &fd, error)) - return FALSE; - if (fd != -1) + /* Testing aid for https://github.com/ostreedev/ostree/issues/2154 */ + const gboolean no_dtb = (sysroot->debug_flags & OSTREE_SYSROOT_DEBUG_TEST_NO_DTB) > 0; + if (!no_dtb) { - ret_layout->devicetree_srcpath = g_strdup ("devicetree"); - ret_layout->devicetree_namever = g_strdup_printf ("devicetree-%s", kver); - in = g_unix_input_stream_new (fd, FALSE); - if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error)) + /* Check for /usr/lib/modules/$kver/devicetree first, if it does not + * exist check for /usr/lib/modules/$kver/dtb/ directory. + */ + if (!ot_openat_ignore_enoent (ret_layout->boot_dfd, "devicetree", &fd, error)) return FALSE; - } - else - { - struct stat stbuf; - /* Check for dtb directory */ - if (!glnx_fstatat_allow_noent (ret_layout->boot_dfd, "dtb", &stbuf, 0, error)) - return FALSE; - - if (errno == 0 && S_ISDIR (stbuf.st_mode)) + if (fd != -1) { - /* devicetree_namever set to NULL indicates a complete directory */ - ret_layout->devicetree_srcpath = g_strdup ("dtb"); - ret_layout->devicetree_namever = NULL; - - if (!checksum_dir_recurse(ret_layout->boot_dfd, "dtb", &checksum, cancellable, error)) + ret_layout->devicetree_srcpath = g_strdup ("devicetree"); + ret_layout->devicetree_namever = g_strdup_printf ("devicetree-%s", kver); + in = g_unix_input_stream_new (fd, FALSE); + if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error)) return FALSE; } - } + else + { + struct stat stbuf; + /* Check for dtb directory */ + if (!glnx_fstatat_allow_noent (ret_layout->boot_dfd, "dtb", &stbuf, 0, error)) + return FALSE; + if (errno == 0 && S_ISDIR (stbuf.st_mode)) + { + /* devicetree_namever set to NULL indicates a complete directory */ + ret_layout->devicetree_srcpath = g_strdup ("dtb"); + ret_layout->devicetree_namever = NULL; + + if (!checksum_dir_recurse(ret_layout->boot_dfd, "dtb", &checksum, cancellable, error)) + return FALSE; + } + } + } g_clear_object (&in); glnx_close_fd (&fd); @@ -1336,7 +1341,8 @@ get_kernel_from_tree_legacy_layouts (int deployment_dfd, * initramfs there, so we need to look in /usr/lib/ostree-boot first. */ static gboolean -get_kernel_from_tree (int deployment_dfd, +get_kernel_from_tree (OstreeSysroot *sysroot, + int deployment_dfd, OstreeKernelLayout **out_layout, GCancellable *cancellable, GError **error) @@ -1345,7 +1351,7 @@ get_kernel_from_tree (int deployment_dfd, g_autoptr(OstreeKernelLayout) legacy_layout = NULL; /* First, gather from usr/lib/modules/$kver if it exists */ - if (!get_kernel_from_tree_usrlib_modules (deployment_dfd, &usrlib_modules_layout, cancellable, error)) + if (!get_kernel_from_tree_usrlib_modules (sysroot, deployment_dfd, &usrlib_modules_layout, cancellable, error)) return FALSE; /* Gather the legacy layout */ @@ -1761,7 +1767,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, /* Find the kernel/initramfs/devicetree in the tree */ g_autoptr(OstreeKernelLayout) kernel_layout = NULL; - if (!get_kernel_from_tree (deployment_dfd, &kernel_layout, + if (!get_kernel_from_tree (sysroot, deployment_dfd, &kernel_layout, cancellable, error)) return FALSE; @@ -1771,7 +1777,6 @@ install_deployment_kernel (OstreeSysroot *sysroot, const char *osname = ostree_deployment_get_osname (deployment); const char *bootcsum = ostree_deployment_get_bootcsum (deployment); - g_assert_cmpstr (kernel_layout->bootcsum, ==, bootcsum); g_autofree char *bootcsumdir = g_strdup_printf ("ostree/%s-%s", osname, bootcsum); g_autofree char *bootconfdir = g_strdup_printf ("loader.%d/entries", new_bootversion); g_autofree char *bootconf_name = g_strdup_printf ("ostree-%d-%s.conf", @@ -2711,7 +2716,7 @@ sysroot_initialize_deployment (OstreeSysroot *self, return FALSE; g_autoptr(OstreeKernelLayout) kernel_layout = NULL; - if (!get_kernel_from_tree (deployment_dfd, &kernel_layout, + if (!get_kernel_from_tree (self, deployment_dfd, &kernel_layout, cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index fa1e5336..1af2fd27 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -38,6 +38,7 @@ typedef enum { /* This is a temporary flag until we fully drop the explicit `systemctl start * ostree-finalize-staged.service` so that tests can exercise the new path unit. */ OSTREE_SYSROOT_DEBUG_TEST_STAGED_PATH = 1 << 3, + OSTREE_SYSROOT_DEBUG_TEST_NO_DTB = 1 << 4, /* https://github.com/ostreedev/ostree/issues/2154 */ } OstreeSysrootDebugFlags; typedef enum { diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index b211fea7..e412ea4d 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -190,6 +190,7 @@ ostree_sysroot_init (OstreeSysroot *self) { "test-fifreeze", OSTREE_SYSROOT_DEBUG_TEST_FIFREEZE }, { "no-xattrs", OSTREE_SYSROOT_DEBUG_NO_XATTRS }, { "test-staged-path", OSTREE_SYSROOT_DEBUG_TEST_STAGED_PATH }, + { "no-dtb", OSTREE_SYSROOT_DEBUG_TEST_NO_DTB }, }; self->debug_flags = g_parse_debug_string (g_getenv ("OSTREE_SYSROOT_DEBUG"), diff --git a/tests/test-osupdate-dtb.sh b/tests/test-osupdate-dtb.sh new file mode 100755 index 00000000..9e0c4686 --- /dev/null +++ b/tests/test-osupdate-dtb.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Copyright (C) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..1" + +# Exports OSTREE_SYSROOT so --sysroot not needed. +kver="3.6.0" +modulesdir="usr/lib/modules/${kver}" +setup_os_repository "archive" "syslinux" ${modulesdir} + +cd ${test_tmpdir} +os_repository_new_commit "test" "test with device tree directory" + +devicetree_path=osdata/${modulesdir}/dtb/asoc-board.dtb +devicetree_overlay_path=osdata/${modulesdir}/dtb/overlays/overlay.dtbo + +mkdir -p osdata/${modulesdir}/dtb +echo "a device tree" > ${devicetree_path} +mkdir -p osdata/${modulesdir}/dtb/overlays +echo "a device tree overlay" > ${devicetree_overlay_path} + +${CMD_PREFIX} ostree --repo=testos-repo commit --tree=dir=osdata/ -b testos/buildmaster/x86_64-runtime +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmaster/x86_64-runtime +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo remote add --set=gpg-verify=false testos file://$(pwd)/testos-repo testos/buildmaster/x86_64-runtime +${CMD_PREFIX} env OSTREE_SYSROOT_DEBUG=${OSTREE_SYSROOT_DEBUG},no-dtb ostree admin deploy --os=testos testos:testos/buildmaster/x86_64-runtime +assert_has_file sysroot/boot/ostree/testos-${bootcsum}/vmlinuz-3.6.0 +assert_not_has_file sysroot/boot/ostree/testos-${bootcsum}/dtb/asoc-board.dtb 'a device tree' +assert_streq $(ls sysroot/boot/ostree | wc -l) 1 +assert_streq $(find sysroot/boot/ostree -name '*.dtb' | wc -l) 0 +${CMD_PREFIX} ostree --repo=testos-repo commit --tree=dir=osdata/ -b testos/buildmaster/x86_64-runtime +env OSTREE_SYSROOT_DEBUG=${OSTREE_SYSROOT_DEBUG},no-dtb ${CMD_PREFIX} ostree admin upgrade --os=testos +${CMD_PREFIX} ostree --repo=testos-repo commit --tree=dir=osdata/ -b testos/buildmaster/x86_64-runtime +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_streq $(ls sysroot/boot/ostree | wc -l) 2 +# Note that the bootcsum computed by the test suite doesn't include devicetree +# And currently we end up installing the dtb for the *previous* deployment +# too which is a bug - in the future this should be fixed to assert 1. +assert_streq $(find sysroot/boot/ostree -name '*.dtb' | wc -l) 2 + +echo "ok update with no dtb to dtb" From 9198fa040cbd6474c15591978b81734de06dab3c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 24 Sep 2020 22:12:23 +0000 Subject: [PATCH 34/58] delta: Some minor code style fixups - Remove some unused variables - Switch to declare-and-initialize with others - Fix some indentation from 4 spaces to 2 (GNU style) --- src/libostree/ostree-repo-static-delta-core.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index b16764ae..02d517a1 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -272,8 +272,6 @@ _ostree_repo_static_delta_verify_signature (OstreeRepo *self, char **out_success_message, GError **error) { - g_autoptr(GVariantBuilder) desc_sign_builder = NULL; - g_autoptr(GVariant) delta_meta = NULL; g_autoptr(GVariant) delta = NULL; if (!ot_variant_read_fd (fd, 0, @@ -284,19 +282,19 @@ _ostree_repo_static_delta_verify_signature (OstreeRepo *self, /* Check if there are signatures for signature engine */ const gchar *signature_key = ostree_sign_metadata_key(sign); GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(sign); - delta_meta = g_variant_get_child_value (delta, 2); + g_autoptr(GVariant) delta_meta = g_variant_get_child_value (delta, 2); if (delta_meta == NULL) - return glnx_throw (error, "no metadata in static-delta superblock"); + return glnx_throw (error, "no metadata in static-delta superblock"); g_autoptr(GVariant) signatures = g_variant_lookup_value (delta_meta, signature_key, signature_format); if (!signatures) - return glnx_throw (error, "no signature for '%s' in static-delta superblock", signature_key); + return glnx_throw (error, "no signature for '%s' in static-delta superblock", signature_key); /* Get static delta superblock */ g_autoptr(GVariant) child = g_variant_get_child_value (delta, 1); if (child == NULL) - return glnx_throw (error, "no metadata in static-delta superblock"); + return glnx_throw (error, "no metadata in static-delta superblock"); g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes(child); return ostree_sign_data_verify (sign, signed_data, signatures, out_success_message, error); @@ -329,7 +327,6 @@ ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, GError **error) { g_autofree char *basename = NULL; - g_autoptr(GVariant) delta = NULL; g_autoptr(GVariant) meta = NULL; const char *dir_or_file_path = gs_file_get_path_cached (dir_or_file); @@ -378,6 +375,7 @@ ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, return glnx_throw (error, "Delta signature verification failed"); } + g_autoptr(GVariant) delta = NULL; if (!ot_variant_read_fd (meta_fd, 0, (GVariantType*)OSTREE_STATIC_DELTA_SIGNED_FORMAT, TRUE, &delta, error)) return FALSE; @@ -435,9 +433,8 @@ ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, if (!have_to_commit) { g_autofree char *detached_path = _ostree_get_relative_static_delta_path (from_checksum, to_checksum, "commitmeta"); - g_autoptr(GVariant) detached_data = NULL; - - detached_data = g_variant_lookup_value (metadata, detached_path, G_VARIANT_TYPE("a{sv}")); + g_autoptr(GVariant) detached_data = + g_variant_lookup_value (metadata, detached_path, G_VARIANT_TYPE("a{sv}")); if (detached_data && !ostree_repo_write_commit_detached_metadata (self, to_checksum, detached_data, @@ -1094,7 +1091,6 @@ ostree_repo_static_delta_verify_signature (OstreeRepo *self, char **out_success_message, GError **error) { - g_autoptr(GVariantBuilder) desc_sign_builder = NULL; g_autoptr(GVariant) delta_meta = NULL; glnx_autofd int delta_fd = -1; From 75342035d5902f299be53d7b541724407463d62c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 25 Sep 2020 14:59:45 -0400 Subject: [PATCH 35/58] Makefile-libostree.am: Uncomment BUILDOPT_IS_DEVEL_BUILD conditional We shouldn't have to toggle the conditional itself during release builds. It should only evaluate to true during devel builds. --- Makefile-libostree.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 1d31c4d8..96b9249b 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -182,9 +182,9 @@ libostree_1_la_SOURCES += \ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym -#if BUILDOPT_IS_DEVEL_BUILD +if BUILDOPT_IS_DEVEL_BUILD symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -#endif +endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= EXTRA_DIST += \ From f04e5d047d7cc4c162ff9760d062924522ec2397 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 25 Sep 2020 15:01:09 -0400 Subject: [PATCH 36/58] lib: Minor versioning related fixes Fix/add the `Since` marker to the new static delta APIs, and update the symbol versioning templates/comments. --- src/libostree/libostree-devel.sym | 2 +- src/libostree/libostree-released.sym | 2 ++ src/libostree/ostree-repo-static-delta-core.c | 4 +++- tests/test-symbols.sh | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index a15654c1..f74f0e00 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,7 +17,7 @@ Boston, MA 02111-1307, USA. ***/ -LIBOSTREE_2020.6 { +LIBOSTREE_2020.7 { global: /* Add symbols here, and uncomment the bits in * Makefile-libostree.am to enable this too. diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index aee434cb..e7d985ca 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -620,6 +620,8 @@ global: /* No new symbols in 2020.5 */ +/* No new symbols in 2020.6 */ + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 02d517a1..d83ec8c4 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -317,6 +317,8 @@ _ostree_repo_static_delta_verify_signature (OstreeRepo *self, * The directory must be named with the form "FROM-TO", where both are * checksums, and it must contain a file named "superblock", along with at least * one part. + * + * Since: 2020.7 */ gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, @@ -1082,7 +1084,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self, * Returns: TRUE if the signature of static delta file is valid using the * signature engine provided, FALSE otherwise. * - * Since: 2020.1 + * Since: 2020.7 */ gboolean ostree_repo_static_delta_verify_signature (OstreeRepo *self, diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index a0781618..697d9ad1 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -66,7 +66,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt < Date: Mon, 17 Aug 2020 09:48:16 -0400 Subject: [PATCH 37/58] lib/bootconfig: Add support for multiple initrd keys Prep for actually teaching the rest of the codebase about this. We keep the primary initrd in the `options` hash table for backwards compatibility. --- apidoc/ostree-sections.txt | 2 + src/libostree/libostree-devel.sym | 2 + src/libostree/ostree-bootconfig-parser.c | 73 +++++++++++++++++++++++- src/libostree/ostree-bootconfig-parser.h | 6 ++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 66b42233..c57d7045 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -37,6 +37,8 @@ ostree_bootconfig_parser_write ostree_bootconfig_parser_write_at ostree_bootconfig_parser_set ostree_bootconfig_parser_get +ostree_bootconfig_parser_set_overlay_initrds +ostree_bootconfig_parser_get_overlay_initrds OSTREE_BOOTCONFIG_PARSER OSTREE_IS_BOOTCONFIG_PARSER diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index f74f0e00..e2d31994 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -24,6 +24,8 @@ global: */ ostree_repo_static_delta_execute_offline_with_signature; ostree_repo_static_delta_verify_signature; + ostree_bootconfig_parser_get_overlay_initrds; + ostree_bootconfig_parser_set_overlay_initrds; } LIBOSTREE_2020.4; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-bootconfig-parser.c b/src/libostree/ostree-bootconfig-parser.c index 67f9fb58..a36a4118 100644 --- a/src/libostree/ostree-bootconfig-parser.c +++ b/src/libostree/ostree-bootconfig-parser.c @@ -30,6 +30,9 @@ struct _OstreeBootconfigParser const char *separators; GHashTable *options; + + /* Additional initrds; the primary initrd is in options. */ + char **overlay_initrds; }; typedef GObjectClass OstreeBootconfigParserClass; @@ -50,6 +53,8 @@ ostree_bootconfig_parser_clone (OstreeBootconfigParser *self) GLNX_HASH_TABLE_FOREACH_KV (self->options, const char*, k, const char*, v) g_hash_table_replace (parser->options, g_strdup (k), g_strdup (v)); + parser->overlay_initrds = g_strdupv (self->overlay_initrds); + return parser; } @@ -76,6 +81,8 @@ ostree_bootconfig_parser_parse_at (OstreeBootconfigParser *self, if (!contents) return FALSE; + g_autoptr(GPtrArray) overlay_initrds = NULL; + g_auto(GStrv) lines = g_strsplit (contents, "\n", -1); for (char **iter = lines; *iter; iter++) { @@ -87,8 +94,19 @@ ostree_bootconfig_parser_parse_at (OstreeBootconfigParser *self, items = g_strsplit_set (line, self->separators, 2); if (g_strv_length (items) == 2 && items[0][0] != '\0') { - g_hash_table_insert (self->options, items[0], items[1]); - g_free (items); /* Transfer ownership */ + if (g_str_equal (items[0], "initrd") && + g_hash_table_contains (self->options, "initrd")) + { + if (!overlay_initrds) + overlay_initrds = g_ptr_array_new_with_free_func (g_free); + g_ptr_array_add (overlay_initrds, items[1]); + g_free (items[0]); + } + else + { + g_hash_table_insert (self->options, items[0], items[1]); + } + g_free (items); /* Free container; we stole the elements */ } else { @@ -97,6 +115,12 @@ ostree_bootconfig_parser_parse_at (OstreeBootconfigParser *self, } } + if (overlay_initrds) + { + g_ptr_array_add (overlay_initrds, NULL); + self->overlay_initrds = (char**)g_ptr_array_free (g_steal_pointer (&overlay_initrds), FALSE); + } + self->parsed = TRUE; return TRUE; @@ -127,6 +151,41 @@ ostree_bootconfig_parser_get (OstreeBootconfigParser *self, return g_hash_table_lookup (self->options, key); } +/** + * ostree_bootconfig_parser_set_overlay_initrds: + * @self: Parser + * @initrds: (array zero-terminated=1) (transfer none) (allow-none): Array of overlay + * initrds or %NULL to unset. + * + * These are rendered as additional `initrd` keys in the final bootloader configs. The + * base initrd is part of the primary keys. + * + * Since: 2020.7 + */ +void +ostree_bootconfig_parser_set_overlay_initrds (OstreeBootconfigParser *self, + char **initrds) +{ + g_assert (g_hash_table_contains (self->options, "initrd")); + g_strfreev (self->overlay_initrds); + self->overlay_initrds = g_strdupv (initrds); +} + +/** + * ostree_bootconfig_parser_get_overlay_initrds: + * @self: Parser + * + * Returns: (array zero-terminated=1) (transfer none) (nullable): Array of initrds or %NULL + * if none are set. + * + * Since: 2020.7 + */ +char** +ostree_bootconfig_parser_get_overlay_initrds (OstreeBootconfigParser *self) +{ + return self->overlay_initrds; +} + static void write_key (OstreeBootconfigParser *self, GString *buf, @@ -165,6 +224,15 @@ ostree_bootconfig_parser_write_at (OstreeBootconfigParser *self, } } + /* Write overlay initrds */ + if (self->overlay_initrds && (g_strv_length (self->overlay_initrds) > 0)) + { + /* we should've written the primary initrd already */ + g_assert (g_hash_table_contains (keys_written, "initrd")); + for (char **it = self->overlay_initrds; it && *it; it++) + write_key (self, buf, "initrd", *it); + } + /* Write unknown fields */ GLNX_HASH_TABLE_FOREACH_KV (self->options, const char*, k, const char*, v) { @@ -197,6 +265,7 @@ ostree_bootconfig_parser_finalize (GObject *object) { OstreeBootconfigParser *self = OSTREE_BOOTCONFIG_PARSER (object); + g_strfreev (self->overlay_initrds); g_hash_table_unref (self->options); G_OBJECT_CLASS (ostree_bootconfig_parser_parent_class)->finalize (object); diff --git a/src/libostree/ostree-bootconfig-parser.h b/src/libostree/ostree-bootconfig-parser.h index aec07535..d03c931c 100644 --- a/src/libostree/ostree-bootconfig-parser.h +++ b/src/libostree/ostree-bootconfig-parser.h @@ -73,5 +73,11 @@ _OSTREE_PUBLIC const char *ostree_bootconfig_parser_get (OstreeBootconfigParser *self, const char *key); +_OSTREE_PUBLIC +void ostree_bootconfig_parser_set_overlay_initrds (OstreeBootconfigParser *self, + char **initrds); + +_OSTREE_PUBLIC +char** ostree_bootconfig_parser_get_overlay_initrds (OstreeBootconfigParser *self); G_END_DECLS From 40fea4c44390116095841ee11b4d336195e56330 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 17 Aug 2020 09:48:17 -0400 Subject: [PATCH 38/58] lib/deploy: Add deploy/stage APIs with options And make the `override_kernel_argv` one of those options. This is mostly a mechanical move here, no functional change otherwise. Prep for adding a new option. --- apidoc/ostree-sections.txt | 2 + src/libostree/libostree-devel.sym | 2 + src/libostree/ostree-sysroot-deploy.c | 121 ++++++++++++++++++++------ src/libostree/ostree-sysroot.h | 30 +++++++ 4 files changed, 129 insertions(+), 26 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index c57d7045..4ef396e6 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -547,7 +547,9 @@ ostree_sysroot_write_deployments ostree_sysroot_write_deployments_with_options ostree_sysroot_write_origin_file ostree_sysroot_stage_tree +ostree_sysroot_stage_tree_with_options ostree_sysroot_deploy_tree +ostree_sysroot_deploy_tree_with_options ostree_sysroot_get_merge_deployment ostree_sysroot_query_deployments_for ostree_sysroot_origin_new_from_refspec diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index e2d31994..3f59c399 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -26,6 +26,8 @@ global: ostree_repo_static_delta_verify_signature; ostree_bootconfig_parser_get_overlay_initrds; ostree_bootconfig_parser_set_overlay_initrds; + ostree_sysroot_deploy_tree_with_options; + ostree_sysroot_stage_tree_with_options; } LIBOSTREE_2020.4; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index d97d96a8..9425316f 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -2688,7 +2688,7 @@ sysroot_initialize_deployment (OstreeSysroot *self, const char *osname, const char *revision, GKeyFile *origin, - char **override_kernel_argv, + OstreeSysrootDeployTreeOpts *opts, OstreeDeployment **out_new_deployment, GCancellable *cancellable, GError **error) @@ -2721,7 +2721,7 @@ sysroot_initialize_deployment (OstreeSysroot *self, return FALSE; _ostree_deployment_set_bootcsum (new_deployment, kernel_layout->bootcsum); - _ostree_deployment_set_bootconfig_from_kargs (new_deployment, override_kernel_argv); + _ostree_deployment_set_bootconfig_from_kargs (new_deployment, opts ? opts->override_kernel_argv : NULL); if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, cancellable, error)) @@ -2852,6 +2852,53 @@ sysroot_finalize_deployment (OstreeSysroot *self, return TRUE; } +/** + * ostree_sysroot_deploy_tree_with_options: + * @self: Sysroot + * @osname: (allow-none): osname to use for merge deployment + * @revision: Checksum to add + * @origin: (allow-none): Origin to use for upgrades + * @provided_merge_deployment: (allow-none): Use this deployment for merge path + * @opts: (allow-none): Options + * @out_new_deployment: (out): The new deployment path + * @cancellable: Cancellable + * @error: Error + * + * Check out deployment tree with revision @revision, performing a 3 + * way merge with @provided_merge_deployment for configuration. + * + * When booted into the sysroot, you should use the + * ostree_sysroot_stage_tree() API instead. + * + * Since: 2020.7 + */ +gboolean +ostree_sysroot_deploy_tree_with_options (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *provided_merge_deployment, + OstreeSysrootDeployTreeOpts *opts, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error) +{ + if (!_ostree_sysroot_ensure_writable (self, error)) + return FALSE; + + g_autoptr(OstreeDeployment) deployment = NULL; + if (!sysroot_initialize_deployment (self, osname, revision, origin, opts, + &deployment, cancellable, error)) + return FALSE; + + if (!sysroot_finalize_deployment (self, deployment, provided_merge_deployment, + cancellable, error)) + return FALSE; + + *out_new_deployment = g_steal_pointer (&deployment); + return TRUE; +} + /** * ostree_sysroot_deploy_tree: * @self: Sysroot @@ -2864,11 +2911,9 @@ sysroot_finalize_deployment (OstreeSysroot *self, * @cancellable: Cancellable * @error: Error * - * Check out deployment tree with revision @revision, performing a 3 - * way merge with @provided_merge_deployment for configuration. + * Older version of ostree_sysroot_stage_tree_with_options(). * - * When booted into the sysroot, you should use the - * ostree_sysroot_stage_tree() API instead. + * Since: 2018.5 */ gboolean ostree_sysroot_deploy_tree (OstreeSysroot *self, @@ -2881,20 +2926,10 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - if (!_ostree_sysroot_ensure_writable (self, error)) - return FALSE; - - g_autoptr(OstreeDeployment) deployment = NULL; - if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv, - &deployment, cancellable, error)) - return FALSE; - - if (!sysroot_finalize_deployment (self, deployment, provided_merge_deployment, - cancellable, error)) - return FALSE; - - *out_new_deployment = g_steal_pointer (&deployment); - return TRUE; + OstreeSysrootDeployTreeOpts opts = { .override_kernel_argv = override_kernel_argv }; + return ostree_sysroot_deploy_tree_with_options (self, osname, revision, origin, + provided_merge_deployment, &opts, + out_new_deployment, cancellable, error); } /* Serialize information about a deployment to a variant, used by the staging @@ -2968,8 +3003,7 @@ _ostree_sysroot_deserialize_deployment_from_variant (GVariant *v, * @cancellable: Cancellable * @error: Error * - * Like ostree_sysroot_deploy_tree(), but "finalization" only occurs at OS - * shutdown time. + * Older version of ostree_sysroot_stage_tree_with_options(). * * Since: 2018.5 */ @@ -2983,6 +3017,41 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, OstreeDeployment **out_new_deployment, GCancellable *cancellable, GError **error) +{ + OstreeSysrootDeployTreeOpts opts = { .override_kernel_argv = override_kernel_argv }; + return ostree_sysroot_stage_tree_with_options (self, osname, revision, origin, + merge_deployment, &opts, + out_new_deployment, cancellable, error); +} + + +/** + * ostree_sysroot_stage_tree_with_options: + * @self: Sysroot + * @osname: (allow-none): osname to use for merge deployment + * @revision: Checksum to add + * @origin: (allow-none): Origin to use for upgrades + * @merge_deployment: (allow-none): Use this deployment for merge path + * @opts: Options + * @out_new_deployment: (out): The new deployment path + * @cancellable: Cancellable + * @error: Error + * + * Like ostree_sysroot_deploy_tree(), but "finalization" only occurs at OS + * shutdown time. + * + * Since: 2020.7 + */ +gboolean +ostree_sysroot_stage_tree_with_options (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *merge_deployment, + OstreeSysrootDeployTreeOpts *opts, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error) { if (!_ostree_sysroot_ensure_writable (self, error)) return FALSE; @@ -3014,8 +3083,8 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, } /* OSTREE_SYSROOT_DEBUG_TEST_STAGED_PATH */ g_autoptr(OstreeDeployment) deployment = NULL; - if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv, - &deployment, cancellable, error)) + if (!sysroot_initialize_deployment (self, osname, revision, origin, opts, &deployment, + cancellable, error)) return FALSE; /* Write out the origin file using the sepolicy from the non-merged root for @@ -3050,9 +3119,9 @@ ostree_sysroot_stage_tree (OstreeSysroot *self, g_variant_builder_add (builder, "{sv}", "merge-deployment", serialize_deployment_to_variant (merge_deployment)); - if (override_kernel_argv) + if (opts && opts->override_kernel_argv) g_variant_builder_add (builder, "{sv}", "kargs", - g_variant_new_strv ((const char *const*)override_kernel_argv, -1)); + g_variant_new_strv ((const char *const*)opts->override_kernel_argv, -1)); const char *parent = dirname (strdupa (_OSTREE_SYSROOT_RUNSTATE_STAGED)); if (!glnx_shutil_mkdir_p_at (AT_FDCWD, parent, 0755, cancellable, error)) diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index d9f5a546..45d6d63c 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -186,6 +186,13 @@ gboolean ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, GCancellable *cancellable, GError **error); +typedef struct { + gboolean unused_bools[8]; + int unused_ints[8]; + char **override_kernel_argv; + gpointer unused_ptrs[7]; +} OstreeSysrootDeployTreeOpts; + _OSTREE_PUBLIC gboolean ostree_sysroot_deploy_tree (OstreeSysroot *self, const char *osname, @@ -197,6 +204,17 @@ gboolean ostree_sysroot_deploy_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sysroot_deploy_tree_with_options (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *provided_merge_deployment, + OstreeSysrootDeployTreeOpts *opts, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_stage_tree (OstreeSysroot *self, const char *osname, @@ -208,6 +226,18 @@ gboolean ostree_sysroot_stage_tree (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sysroot_stage_tree_with_options (OstreeSysroot *self, + const char *osname, + const char *revision, + GKeyFile *origin, + OstreeDeployment *merge_deployment, + OstreeSysrootDeployTreeOpts *opts, + OstreeDeployment **out_new_deployment, + GCancellable *cancellable, + GError **error); + + _OSTREE_PUBLIC gboolean ostree_sysroot_deployment_set_mutable (OstreeSysroot *self, OstreeDeployment *deployment, From 81b13da8e35e18cfb445f66bf365420aa0e9fa0a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 17 Aug 2020 09:48:18 -0400 Subject: [PATCH 39/58] lib/deploy: Add support for overlay initrds In FCOS and RHCOS, the need to configure software in the initramfs has come up multiple times. Sometimes, using kernel arguments suffices. Other times, it really must be a configuration file. Rebuilding the initramfs on the client-side however is a costly operation. Not only does it add complexity to the update workflow, it also erodes a lot of the value obtained from using the baked "blessed" initramfs from the tree itself. One elegant way to address this is to allow specifying multiple initramfses. This is supported by most bootloaders (notably GRUB) and results in each initrd being overlayed on top of each other. This patch allows libostree clients to leverage this so that they can avoid regenerating the initramfs entirely. libostree itself is agnostic as to what kind and how much data overlay initrds contain. It's up to the clients to enforce such boundaries. To implement this, we add a new ostree_sysroot_stage_overlay_initrd which takes a file descriptor and returns a checksum. Then users can pass these checksums when calling the deploy APIs via the new array option `overlay_initrds`. We copy these files into `/boot` and add them to the BLS as another `initrd` entry. --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-deployment-private.h | 9 ++ src/libostree/ostree-deployment.c | 32 +++++ src/libostree/ostree-sysroot-cleanup.c | 43 +++++++ src/libostree/ostree-sysroot-deploy.c | 112 ++++++++++++++++++ src/libostree/ostree-sysroot-private.h | 4 + src/libostree/ostree-sysroot.c | 22 ++++ src/libostree/ostree-sysroot.h | 10 +- src/ostree/ot-admin-builtin-deploy.c | 68 +++++++++-- tests/kolainst/destructive/overlay-initrds.sh | 96 +++++++++++++++ 11 files changed, 390 insertions(+), 8 deletions(-) create mode 100755 tests/kolainst/destructive/overlay-initrds.sh diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 4ef396e6..c1d6b35e 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -548,6 +548,7 @@ ostree_sysroot_write_deployments_with_options ostree_sysroot_write_origin_file ostree_sysroot_stage_tree ostree_sysroot_stage_tree_with_options +ostree_sysroot_stage_overlay_initrd ostree_sysroot_deploy_tree ostree_sysroot_deploy_tree_with_options ostree_sysroot_get_merge_deployment diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 3f59c399..341a22a8 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -28,6 +28,7 @@ global: ostree_bootconfig_parser_set_overlay_initrds; ostree_sysroot_deploy_tree_with_options; ostree_sysroot_stage_tree_with_options; + ostree_sysroot_stage_overlay_initrd; } LIBOSTREE_2020.4; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-deployment-private.h b/src/libostree/ostree-deployment-private.h index ad77317d..b339ae26 100644 --- a/src/libostree/ostree-deployment-private.h +++ b/src/libostree/ostree-deployment-private.h @@ -37,6 +37,8 @@ G_BEGIN_DECLS * @origin: How to construct an upgraded version of this tree * @unlocked: The unlocked state * @staged: TRUE iff this deployment is staged + * @overlay_initrds: Checksums of staged additional initrds for this deployment + * @overlay_initrds_id: Unique ID generated from initrd checksums; used to compare deployments */ struct _OstreeDeployment { @@ -52,8 +54,15 @@ struct _OstreeDeployment GKeyFile *origin; OstreeDeploymentUnlockedState unlocked; gboolean staged; + char **overlay_initrds; + char *overlay_initrds_id; }; void _ostree_deployment_set_bootcsum (OstreeDeployment *self, const char *bootcsum); +void _ostree_deployment_set_overlay_initrds (OstreeDeployment *self, + char **overlay_initrds); + +char** _ostree_deployment_get_overlay_initrds (OstreeDeployment *self); + G_END_DECLS diff --git a/src/libostree/ostree-deployment.c b/src/libostree/ostree-deployment.c index 70e1bc49..182bceea 100644 --- a/src/libostree/ostree-deployment.c +++ b/src/libostree/ostree-deployment.c @@ -158,6 +158,34 @@ _ostree_deployment_set_bootcsum (OstreeDeployment *self, self->bootcsum = g_strdup (bootcsum); } +void +_ostree_deployment_set_overlay_initrds (OstreeDeployment *self, + char **overlay_initrds) +{ + g_clear_pointer (&self->overlay_initrds, g_strfreev); + g_clear_pointer (&self->overlay_initrds_id, g_free); + + if (!overlay_initrds || g_strv_length (overlay_initrds) == 0) + return; + + /* Generate a unique ID representing this combination of overlay initrds. This is so that + * ostree_sysroot_write_deployments_with_options() can easily compare initrds when + * comparing deployments for whether a bootswap is necessary. We could be fancier here but + * meh... this works. */ + g_autoptr(GString) id = g_string_new (NULL); + for (char **it = overlay_initrds; it && *it; it++) + g_string_append (id, *it); + + self->overlay_initrds = g_strdupv (overlay_initrds); + self->overlay_initrds_id = g_string_free (g_steal_pointer (&id), FALSE); +} + +char** +_ostree_deployment_get_overlay_initrds (OstreeDeployment *self) +{ + return self->overlay_initrds; +} + /** * ostree_deployment_clone: * @self: Deployment @@ -175,6 +203,8 @@ ostree_deployment_clone (OstreeDeployment *self) new_bootconfig = ostree_bootconfig_parser_clone (self->bootconfig); ostree_deployment_set_bootconfig (ret, new_bootconfig); + _ostree_deployment_set_overlay_initrds (ret, self->overlay_initrds); + if (self->origin) { g_autoptr(GKeyFile) new_origin = NULL; @@ -238,6 +268,8 @@ ostree_deployment_finalize (GObject *object) g_free (self->bootcsum); g_clear_object (&self->bootconfig); g_clear_pointer (&self->origin, g_key_file_unref); + g_strfreev (self->overlay_initrds); + g_free (self->overlay_initrds_id); G_OBJECT_CLASS (ostree_deployment_parent_class)->finalize (object); } diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index ffad4130..27122834 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -298,6 +298,8 @@ cleanup_old_deployments (OstreeSysroot *self, g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); g_autoptr(GHashTable) active_boot_checksums = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); + g_autoptr(GHashTable) active_overlay_initrds = + g_hash_table_new (g_str_hash, g_str_equal); /* borrows from deployment's bootconfig */ for (guint i = 0; i < self->deployments->len; i++) { OstreeDeployment *deployment = self->deployments->pdata[i]; @@ -306,6 +308,11 @@ cleanup_old_deployments (OstreeSysroot *self, /* Transfer ownership */ g_hash_table_replace (active_deployment_dirs, deployment_path, deployment_path); g_hash_table_replace (active_boot_checksums, bootcsum, bootcsum); + + OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (deployment); + char **initrds = ostree_bootconfig_parser_get_overlay_initrds (bootconfig); + for (char **it = initrds; it && *it; it++) + g_hash_table_add (active_overlay_initrds, (char*)glnx_basename (*it)); } /* Find all deployment directories, both active and inactive */ @@ -349,6 +356,42 @@ cleanup_old_deployments (OstreeSysroot *self, return FALSE; } + /* Clean up overlay initrds */ + glnx_autofd int overlays_dfd = + glnx_opendirat_with_errno (self->sysroot_fd, _OSTREE_SYSROOT_INITRAMFS_OVERLAYS, FALSE); + if (overlays_dfd < 0) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "open(initrd_overlays)"); + } + else + { + g_autoptr(GPtrArray) initrds_to_delete = g_ptr_array_new_with_free_func (g_free); + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + if (!glnx_dirfd_iterator_init_at (overlays_dfd, ".", TRUE, &dfd_iter, error)) + return FALSE; + while (TRUE) + { + struct dirent *dent; + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + + /* there shouldn't be other file types there, but let's be conservative */ + if (dent->d_type != DT_REG) + continue; + + if (!g_hash_table_lookup (active_overlay_initrds, dent->d_name)) + g_ptr_array_add (initrds_to_delete, g_strdup (dent->d_name)); + } + for (guint i = 0; i < initrds_to_delete->len; i++) + { + if (!ot_ensure_unlinked_at (overlays_dfd, initrds_to_delete->pdata[i], error)) + return FALSE; + } + } + return TRUE; } diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 9425316f..1c4fb5dc 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1859,6 +1859,47 @@ install_deployment_kernel (OstreeSysroot *sysroot, } } + g_autoptr(GPtrArray) overlay_initrds = NULL; + for (char **it = _ostree_deployment_get_overlay_initrds (deployment); it && *it; it++) + { + char *checksum = *it; + + /* Overlay initrds are not part of the bootcsum dir; they're not part of the tree + * proper. Instead they're in /boot/ostree/initramfs-overlays/ named by their csum. + * Doing it this way allows sharing the same bootcsum dir for multiple deployments + * with the only change being in overlay initrds (or conversely, the same overlay + * across different boocsums). Eventually, it'd be nice to have an OSTree repo in + * /boot itself and drop the boocsum dir concept entirely. */ + + g_autofree char *destpath = + g_strdup_printf ("/" _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS "/%s.img", checksum); + const char *rel_destpath = destpath + 1; + + /* lazily allocate array and create dir so we don't pollute /boot if not needed */ + if (overlay_initrds == NULL) + { + overlay_initrds = g_ptr_array_new_with_free_func (g_free); + + if (!glnx_shutil_mkdir_p_at (boot_dfd, _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS, + 0755, cancellable, error)) + return FALSE; + } + + if (!glnx_fstatat_allow_noent (boot_dfd, rel_destpath, NULL, 0, error)) + return FALSE; + if (errno == ENOENT) + { + g_autofree char *srcpath = + g_strdup_printf (_OSTREE_SYSROOT_RUNSTATE_STAGED_INITRDS_DIR "/%s", checksum); + if (!install_into_boot (repo, sepolicy, AT_FDCWD, srcpath, boot_dfd, rel_destpath, + cancellable, error)) + return FALSE; + } + + /* these are used lower down to populate the bootconfig */ + g_ptr_array_add (overlay_initrds, g_steal_pointer (&destpath)); + } + g_autofree char *contents = NULL; if (!glnx_fstatat_allow_noent (deployment_dfd, "usr/lib/os-release", &stbuf, 0, error)) return FALSE; @@ -1938,6 +1979,12 @@ install_deployment_kernel (OstreeSysroot *sysroot, g_autofree char * initrd_boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->initramfs_namever, NULL); ostree_bootconfig_parser_set (bootconfig, "initrd", initrd_boot_relpath); + + if (overlay_initrds) + { + g_ptr_array_add (overlay_initrds, NULL); + ostree_bootconfig_parser_set_overlay_initrds (bootconfig, (char**)overlay_initrds->pdata); + } } else { @@ -2135,6 +2182,10 @@ deployment_bootconfigs_equal (OstreeRepo *repo, if (strcmp (a_bootcsum, b_bootcsum) != 0) return FALSE; + /* same initrd overlays? */ + if (g_strcmp0 (a->overlay_initrds_id, b->overlay_initrds_id) != 0) + return FALSE; + /* same kargs? */ g_autofree char *a_boot_options_without_ostree = get_deployment_nonostree_kargs (a); g_autofree char *b_boot_options_without_ostree = get_deployment_nonostree_kargs (b); @@ -2722,6 +2773,7 @@ sysroot_initialize_deployment (OstreeSysroot *self, _ostree_deployment_set_bootcsum (new_deployment, kernel_layout->bootcsum); _ostree_deployment_set_bootconfig_from_kargs (new_deployment, opts ? opts->override_kernel_argv : NULL); + _ostree_deployment_set_overlay_initrds (new_deployment, opts ? opts->overlay_initrds : NULL); if (!prepare_deployment_etc (self, repo, new_deployment, deployment_dfd, cancellable, error)) @@ -2991,6 +3043,63 @@ _ostree_sysroot_deserialize_deployment_from_variant (GVariant *v, } +/** + * ostree_sysroot_stage_overlay_initrd: + * @self: Sysroot + * @fd: (transfer none): File descriptor to overlay initrd + * @out_checksum: (out) (transfer full): Overlay initrd checksum + * @cancellable: Cancellable + * @error: Error + * + * Stage an overlay initrd to be used in an upcoming deployment. Returns a checksum which + * can be passed to ostree_sysroot_deploy_tree_with_options() or + * ostree_sysroot_stage_tree_with_options() via the `overlay_initrds` array option. + * + * Since: 2020.7 + */ +gboolean +ostree_sysroot_stage_overlay_initrd (OstreeSysroot *self, + int fd, + char **out_checksum, + GCancellable *cancellable, + GError **error) +{ + g_return_val_if_fail (fd != -1, FALSE); + g_return_val_if_fail (out_checksum != NULL, FALSE); + + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED_INITRDS_DIR, + 0755, cancellable, error)) + return FALSE; + + glnx_autofd int staged_initrds_dfd = -1; + if (!glnx_opendirat (AT_FDCWD, _OSTREE_SYSROOT_RUNSTATE_STAGED_INITRDS_DIR, FALSE, + &staged_initrds_dfd, error)) + return FALSE; + + g_auto(GLnxTmpfile) overlay_initrd = { 0, }; + if (!glnx_open_tmpfile_linkable_at (staged_initrds_dfd, ".", O_WRONLY | O_CLOEXEC, + &overlay_initrd, error)) + return FALSE; + + char checksum[_OSTREE_SHA256_STRING_LEN+1]; + { + g_autoptr(GOutputStream) output = g_unix_output_stream_new (overlay_initrd.fd, FALSE); + g_autoptr(GInputStream) input = g_unix_input_stream_new (fd, FALSE); + g_autofree guchar *digest = NULL; + if (!ot_gio_splice_get_checksum (output, input, &digest, cancellable, error)) + return FALSE; + ot_bin2hex (checksum, (guint8*)digest, _OSTREE_SHA256_DIGEST_LEN); + } + + if (!glnx_link_tmpfile_at (&overlay_initrd, GLNX_LINK_TMPFILE_REPLACE, + staged_initrds_dfd, checksum, error)) + return FALSE; + + *out_checksum = g_strdup (checksum); + return TRUE; +} + + /** * ostree_sysroot_stage_tree: * @self: Sysroot @@ -3122,6 +3231,9 @@ ostree_sysroot_stage_tree_with_options (OstreeSysroot *self, if (opts && opts->override_kernel_argv) g_variant_builder_add (builder, "{sv}", "kargs", g_variant_new_strv ((const char *const*)opts->override_kernel_argv, -1)); + if (opts && opts->overlay_initrds) + g_variant_builder_add (builder, "{sv}", "overlay-initrds", + g_variant_new_strv ((const char *const*)opts->overlay_initrds, -1)); const char *parent = dirname (strdupa (_OSTREE_SYSROOT_RUNSTATE_STAGED)); if (!glnx_shutil_mkdir_p_at (AT_FDCWD, parent, 0755, cancellable, error)) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 1af2fd27..318b0b19 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -84,10 +84,14 @@ struct OstreeSysroot { /* We keep some transient state in /run */ #define _OSTREE_SYSROOT_RUNSTATE_STAGED "/run/ostree/staged-deployment" #define _OSTREE_SYSROOT_RUNSTATE_STAGED_LOCKED "/run/ostree/staged-deployment-locked" +#define _OSTREE_SYSROOT_RUNSTATE_STAGED_INITRDS_DIR "/run/ostree/staged-initrds/" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_DIR "/run/ostree/deployment-state/" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT "unlocked-development" #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_TRANSIENT "unlocked-transient" +#define _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS "ostree/initramfs-overlays" +#define _OSTREE_SYSROOT_INITRAMFS_OVERLAYS "boot/" _OSTREE_SYSROOT_BOOT_INITRAMFS_OVERLAYS + gboolean _ostree_sysroot_ensure_writable (OstreeSysroot *self, GError **error); diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index e412ea4d..e0813b55 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -815,6 +815,24 @@ list_deployments_process_one_boot_entry (OstreeSysroot *self, return FALSE; ostree_deployment_set_bootconfig (deployment, config); + char **overlay_initrds = ostree_bootconfig_parser_get_overlay_initrds (config); + g_autoptr(GPtrArray) initrds_chksums = NULL; + for (char **it = overlay_initrds; it && *it; it++) + { + const char *basename = glnx_basename (*it); + if (strlen (basename) != (_OSTREE_SHA256_STRING_LEN + strlen (".img"))) + return glnx_throw (error, "Malformed overlay initrd filename: %s", basename); + + if (!initrds_chksums) /* lazy init */ + initrds_chksums = g_ptr_array_new_full (g_strv_length (overlay_initrds), g_free); + g_ptr_array_add (initrds_chksums, g_strndup (basename, _OSTREE_SHA256_STRING_LEN)); + } + + if (initrds_chksums) + { + g_ptr_array_add (initrds_chksums, NULL); + _ostree_deployment_set_overlay_initrds (deployment, (char**)initrds_chksums->pdata); + } g_ptr_array_add (inout_deployments, g_object_ref (deployment)); return TRUE; @@ -967,8 +985,10 @@ _ostree_sysroot_reload_staged (OstreeSysroot *self, /* Parse it */ g_autoptr(GVariant) target = NULL; g_autofree char **kargs = NULL; + g_autofree char **overlay_initrds = NULL; g_variant_dict_lookup (staged_deployment_dict, "target", "@a{sv}", &target); g_variant_dict_lookup (staged_deployment_dict, "kargs", "^a&s", &kargs); + g_variant_dict_lookup (staged_deployment_dict, "overlay-initrds", "^a&s", &overlay_initrds); if (target) { g_autoptr(OstreeDeployment) staged = @@ -980,6 +1000,8 @@ _ostree_sysroot_reload_staged (OstreeSysroot *self, if (!load_origin (self, staged, NULL, error)) return FALSE; + _ostree_deployment_set_overlay_initrds (staged, overlay_initrds); + self->staged_deployment = g_steal_pointer (&staged); self->staged_deployment_data = g_steal_pointer (&staged_deployment_data); /* We set this flag for ostree_deployment_is_staged() because that API diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index 45d6d63c..3a3b6a77 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -186,11 +186,19 @@ gboolean ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sysroot_stage_overlay_initrd (OstreeSysroot *self, + int fd, + char **out_checksum, + GCancellable *cancellable, + GError **error); + typedef struct { gboolean unused_bools[8]; int unused_ints[8]; char **override_kernel_argv; - gpointer unused_ptrs[7]; + char **overlay_initrds; + gpointer unused_ptrs[6]; } OstreeSysrootDeployTreeOpts; _OSTREE_PUBLIC diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index bcece3f6..8156cc15 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -44,6 +44,7 @@ static gboolean opt_kernel_proc_cmdline; static char *opt_osname; static char *opt_origin_path; static gboolean opt_kernel_arg_none; +static char **opt_overlay_initrds; static GOptionEntry options[] = { { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, @@ -59,6 +60,7 @@ static GOptionEntry options[] = { { "karg", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_kernel_argv, "Set kernel argument, like root=/dev/sda1; this overrides any earlier argument with the same name", "NAME=VALUE" }, { "karg-append", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_kernel_argv_append, "Append kernel argument; useful with e.g. console= that can be used multiple times", "NAME=VALUE" }, { "karg-none", 0, 0, G_OPTION_ARG_NONE, &opt_kernel_arg_none, "Do not import kernel arguments", NULL }, + { "overlay-initrd", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_overlay_initrds, "Overlay iniramfs file", "FILE" }, { NULL } }; @@ -167,24 +169,76 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat ostree_kernel_args_append_argv (kargs, opt_kernel_argv_append); } - g_autoptr(OstreeDeployment) new_deployment = NULL; + g_autoptr(GPtrArray) overlay_initrd_chksums = NULL; + for (char **it = opt_overlay_initrds; it && *it; it++) + { + const char *path = *it; + + glnx_autofd int fd = -1; + if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error)) + return FALSE; + + g_autofree char *chksum = NULL; + if (!ostree_sysroot_stage_overlay_initrd (sysroot, fd, &chksum, cancellable, error)) + return FALSE; + + if (!overlay_initrd_chksums) + overlay_initrd_chksums = g_ptr_array_new_full (g_strv_length (opt_overlay_initrds), g_free); + g_ptr_array_add (overlay_initrd_chksums, g_steal_pointer (&chksum)); + } + + if (overlay_initrd_chksums) + g_ptr_array_add (overlay_initrd_chksums, NULL); + g_auto(GStrv) kargs_strv = kargs ? ostree_kernel_args_to_strv (kargs) : NULL; + + OstreeSysrootDeployTreeOpts opts = { + .override_kernel_argv = kargs_strv, + .overlay_initrds = overlay_initrd_chksums ? (char**)overlay_initrd_chksums->pdata : NULL, + }; + + g_autoptr(OstreeDeployment) new_deployment = NULL; if (opt_stage) { if (opt_retain_pending || opt_retain_rollback) return glnx_throw (error, "--stage cannot currently be combined with --retain arguments"); if (opt_not_as_default) return glnx_throw (error, "--stage cannot currently be combined with --not-as-default"); - if (!ostree_sysroot_stage_tree (sysroot, opt_osname, revision, origin, merge_deployment, - kargs_strv, &new_deployment, cancellable, error)) - return FALSE; + /* use old API if we can to exercise it in CI */ + if (!overlay_initrd_chksums) + { + if (!ostree_sysroot_stage_tree (sysroot, opt_osname, revision, origin, + merge_deployment, kargs_strv, &new_deployment, + cancellable, error)) + return FALSE; + } + else + { + if (!ostree_sysroot_stage_tree_with_options (sysroot, opt_osname, revision, + origin, merge_deployment, &opts, + &new_deployment, cancellable, error)) + return FALSE; + } g_assert (new_deployment); } else { - if (!ostree_sysroot_deploy_tree (sysroot, opt_osname, revision, origin, merge_deployment, - kargs_strv, &new_deployment, cancellable, error)) - return FALSE; + /* use old API if we can to exercise it in CI */ + if (!overlay_initrd_chksums) + { + if (!ostree_sysroot_deploy_tree (sysroot, opt_osname, revision, origin, + merge_deployment, kargs_strv, &new_deployment, + cancellable, error)) + return FALSE; + } + else + { + if (!ostree_sysroot_deploy_tree_with_options (sysroot, opt_osname, revision, + origin, merge_deployment, &opts, + &new_deployment, cancellable, + error)) + return FALSE; + } g_assert (new_deployment); OstreeSysrootSimpleWriteDeploymentFlags flags = OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN; diff --git a/tests/kolainst/destructive/overlay-initrds.sh b/tests/kolainst/destructive/overlay-initrds.sh new file mode 100755 index 00000000..b24d2d08 --- /dev/null +++ b/tests/kolainst/destructive/overlay-initrds.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# https://github.com/ostreedev/ostree/issues/1667 +set -xeuo pipefail + +. ${KOLA_EXT_DATA}/libinsttest.sh + +# we don't just use `rpm-ostree initramfs-etc` here because we want to be able +# to test more things + +# dracut prints all the cmdline args, including those from /etc/cmdline.d, so +# the way we test that an initrd was included is to just add kargs there and +# grep for it +create_initrd_with_dracut_karg() { + local karg=$1; shift + local d + d=$(mktemp -dp /var/tmp) + mkdir -p "${d}/etc/cmdline.d" + echo "${karg}" > "${d}/etc/cmdline.d/${karg}.conf" + echo "etc/cmdline.d/${karg}.conf" | \ + cpio -D "${d}" -o -H newc --reproducible > "/var/tmp/${karg}.img" +} + +check_for_dracut_karg() { + local karg=$1; shift + # https://github.com/dracutdevs/dracut/blob/38ea7e821b/modules.d/98dracut-systemd/dracut-cmdline.sh#L17 + journalctl -b 0 -t dracut-cmdline \ + --grep "Using kernel command line parameters:.* ${karg} " +} + +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") + create_initrd_with_dracut_karg ostree.test1 + # let's use the deploy API first + ostree admin deploy "${host_refspec}" \ + --overlay-initrd /var/tmp/ostree.test1.img + /tmp/autopkgtest-reboot "2" + ;; + 2) + # verify that ostree.test1 is here + check_for_dracut_karg ostree.test1 + img_sha=$(sha256sum < /var/tmp/ostree.test1.img | cut -f 1 -d ' ') + test -f "/boot/ostree/initramfs-overlays/${img_sha}.img" + + # now let's change to ostree.test2 + create_initrd_with_dracut_karg ostree.test2 + + # let's use the staging API this time + ostree admin deploy "${host_refspec}" --stage \ + --overlay-initrd /var/tmp/ostree.test2.img + /tmp/autopkgtest-reboot "3" + ;; + 3) + # verify that ostree.test1 is gone, but ostree.test2 is here + if check_for_dracut_karg ostree.test1; then + assert_not_reached "Unexpected ostree.test1 karg found" + fi + check_for_dracut_karg ostree.test2 + + # both the new and old initrds should still be there since they're + # referenced in the BLS + test1_sha=$(sha256sum < /var/tmp/ostree.test1.img | cut -f 1 -d ' ') + test2_sha=$(sha256sum < /var/tmp/ostree.test2.img | cut -f 1 -d ' ') + test -f "/boot/ostree/initramfs-overlays/${test1_sha}.img" + test -f "/boot/ostree/initramfs-overlays/${test2_sha}.img" + + # OK, now let's deploy an identical copy of this test + ostree admin deploy "${host_refspec}" \ + --overlay-initrd /var/tmp/ostree.test2.img + + # Now the deployment with ostree.test1 should've been GC'ed; check that its + # initrd was cleaned up + test ! -f "/boot/ostree/initramfs-overlays/${test1_sha}.img" + test -f "/boot/ostree/initramfs-overlays/${test2_sha}.img" + + # deploy again to check that no bootconfig swap was needed; this verifies + # that deployment overlay initrds can be successfully compared + ostree admin deploy "${host_refspec}" \ + --overlay-initrd /var/tmp/ostree.test2.img |& tee /tmp/out.txt + assert_file_has_content /tmp/out.txt 'bootconfig swap: no' + + # finally, let's check that we can overlay multiple initrds + ostree admin deploy "${host_refspec}" --stage \ + --overlay-initrd /var/tmp/ostree.test1.img \ + --overlay-initrd /var/tmp/ostree.test2.img + /tmp/autopkgtest-reboot "4" + ;; + 4) + check_for_dracut_karg ostree.test1 + check_for_dracut_karg ostree.test2 + test1_sha=$(sha256sum < /var/tmp/ostree.test1.img | cut -f 1 -d ' ') + test2_sha=$(sha256sum < /var/tmp/ostree.test2.img | cut -f 1 -d ' ') + test -f "/boot/ostree/initramfs-overlays/${test1_sha}.img" + test -f "/boot/ostree/initramfs-overlays/${test2_sha}.img" + ;; + *) fatal "Unexpected AUTOPKGTEST_REBOOT_MARK=${AUTOPKGTEST_REBOOT_MARK}" ;; +esac From 206f1d3a13189e55027ffadd0b8b434768c3ec0f Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 10 Aug 2020 12:05:06 +0100 Subject: [PATCH 40/58] lib/repo: Add mode and tombstone config options to the summary file Currently, they are set in the `config` file and cause that to be downloaded on every pull. Given that the client is already pulling the `summary` file, it makes sense to avoid an additional network round trip and cache those options in the `summary` file. Signed-off-by: Philip Withnall Helps: #2165 --- src/libostree/ostree-repo-private.h | 2 ++ src/libostree/ostree-repo.c | 18 ++++++++++++++++++ src/ostree/ot-dump.c | 16 ++++++++++++++++ tests/test-summary-view.sh | 2 +- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 8c1f5071..a48feca4 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -55,6 +55,8 @@ G_BEGIN_DECLS #define OSTREE_SUMMARY_EXPIRES "ostree.summary.expires" #define OSTREE_SUMMARY_COLLECTION_ID "ostree.summary.collection-id" #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map" +#define OSTREE_SUMMARY_MODE "ostree.summary.mode" +#define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits" #define _OSTREE_PAYLOAD_LINK_PREFIX "../" #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 4283f68e..ba3e877f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5816,6 +5816,24 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); } + { + g_autofree char *remote_mode_str = NULL; + if (!ot_keyfile_get_value_with_default (self->config, "core", "mode", "bare", + &remote_mode_str, error)) + return FALSE; + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_MODE, + g_variant_new_string (remote_mode_str)); + } + + { + gboolean tombstone_commits = FALSE; + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "tombstone-commits", FALSE, + &tombstone_commits, error)) + return FALSE; + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_TOMBSTONE_COMMITS, + g_variant_new_boolean (tombstone_commits)); + } + /* Add refs which have a collection specified, which could be in refs/mirrors, * refs/heads, and/or refs/remotes. */ { diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 225d1845..1f911d4e 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -361,6 +361,22 @@ ot_dump_summary_bytes (GBytes *summary_bytes, pretty_key = "Collection Map"; value_str = g_strdup ("(printed above)"); } + else if (g_strcmp0 (key, OSTREE_SUMMARY_MODE) == 0) + { + OstreeRepoMode repo_mode; + const char *repo_mode_str = g_variant_get_string (value, NULL); + + pretty_key = "Repository Mode"; + if (!ostree_repo_mode_from_string (repo_mode_str, &repo_mode, NULL)) + value_str = g_strdup_printf ("Invalid (‘%s’)", repo_mode_str); + else + value_str = g_strdup (repo_mode_str); + } + else if (g_strcmp0 (key, OSTREE_SUMMARY_TOMBSTONE_COMMITS) == 0) + { + pretty_key = "Has Tombstone Commits"; + value_str = g_strdup (g_variant_get_boolean (value) ? "Yes" : "No"); + } else { value_str = g_variant_print (value, FALSE); diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh index 14de0294..f6278a85 100755 --- a/tests/test-summary-view.sh +++ b/tests/test-summary-view.sh @@ -64,5 +64,5 @@ echo "ok view summary" ${OSTREE} summary --raw > raw-summary.txt assert_file_has_content_literal raw-summary.txt "('main', (" assert_file_has_content_literal raw-summary.txt "('other', (" -assert_file_has_content_literal raw-summary.txt "{'ostree.summary.last-modified': Date: Mon, 10 Aug 2020 12:06:35 +0100 Subject: [PATCH 41/58] lib/pull: Read mode and tombstone options from summary file if possible Otherwise, fall back to downloading and reading them from the `config` file. See the previous commit for details. Signed-off-by: Philip Withnall Fixes: #2165 --- src/libostree/ostree-repo-pull.c | 68 +++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a6401907..f16ccec5 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2001,6 +2001,8 @@ start_fetch (OtPullData *pull_data, is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch); } +/* Deprecated: code should load options from the `summary` file rather than + * downloading the remote’s `config` file, to save on network round trips. */ static gboolean load_remote_repo_config (OtPullData *pull_data, GKeyFile **out_keyfile, @@ -3757,30 +3759,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error)) goto out; } - else - { - if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) - goto out; - - if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", - &remote_mode_str, error)) - goto out; - - if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) - goto out; - - if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "tombstone-commits", FALSE, - &pull_data->has_tombstone_commits, error)) - goto out; - - if (pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't pull from archives with mode \"%s\"", - remote_mode_str); - goto out; - } - } } /* Change some option defaults if we're actually pulling from a local @@ -3854,6 +3832,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GVariant) deltas = NULL; g_autoptr(GVariant) additional_metadata = NULL; gboolean summary_from_cache = FALSE; + gboolean remote_mode_loaded = FALSE; + gboolean tombstone_commits = FALSE; if (summary_sig_bytes_v) { @@ -4167,6 +4147,46 @@ ostree_repo_pull_with_options (OstreeRepo *self, csum_data); } } + + if (pull_data->summary && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits)) + { + if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) + goto out; + pull_data->has_tombstone_commits = tombstone_commits; + remote_mode_loaded = TRUE; + } + else if (pull_data->remote_repo_local == NULL) + { + /* Fall-back path which loads the necessary config from the remote’s + * `config` file. Doing so is deprecated since it means an + * additional round trip to the remote for each pull. No need to do + * it for local pulls. */ + if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) + goto out; + + if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", + &remote_mode_str, error)) + goto out; + + if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) + goto out; + + if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "tombstone-commits", FALSE, + &pull_data->has_tombstone_commits, error)) + goto out; + + remote_mode_loaded = TRUE; + } + + if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't pull from archives with mode \"%s\"", + remote_mode_str); + goto out; + } } if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) From 23bdc4e5df41a1f53ce387f90eaf6b79c644b2da Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 10 Aug 2020 12:07:22 +0100 Subject: [PATCH 42/58] ostree/dump: Fix a memory leak Re-using the `refs` variable for the main list of refs, plus the iterated lists, meant that the main list was never freed (although all the iterated ones were freed correctly). Fix this by using two variables rather than reusing the one. Signed-off-by: Philip Withnall --- src/ostree/ot-dump.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 1f911d4e..a8ed54a2 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -322,10 +322,11 @@ ot_dump_summary_bytes (GBytes *summary_bytes, collection_map = g_variant_lookup_value (exts, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); if (collection_map != NULL) { + g_autoptr(GVariant) collection_refs = NULL; g_variant_iter_init (&iter, collection_map); - while (g_variant_iter_loop (&iter, "{&s@a(s(taya{sv}))}", &collection_id, &refs)) - dump_summary_refs (collection_id, refs); + while (g_variant_iter_loop (&iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) + dump_summary_refs (collection_id, collection_refs); } /* Print out the additional metadata. */ From 015d44d58d7ed0f9a4d6b0100f77ecf402b4b895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Mon, 28 Sep 2020 16:55:03 +0200 Subject: [PATCH 43/58] docs: Add Jekyll and theme config --- docs/_config.yml | 40 ++++++++++++++++++++++++++++ docs/_sass/color_schemes/coreos.scss | 1 + 2 files changed, 41 insertions(+) create mode 100644 docs/_config.yml create mode 100644 docs/_sass/color_schemes/coreos.scss diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 00000000..2070f1cf --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,40 @@ +title: ostreedev/ostree +description: ostree documentation +baseurl: "/ostree" +url: "https://coreos.github.io" +# Comment above and use below for local development +# url: "http://localhost:4000" +permalink: /:title/ +markdown: kramdown + +remote_theme: coreos/just-the-docs +plugins: + - jekyll-remote-theme + +color_scheme: coreos + +# Aux links for the upper right navigation +aux_links: + "OSTree on GitHub": + - "https://github.com/ostreedev/ostree" + +footer_content: "Copyright © Red Hat, Inc. and others." + +# Footer last edited timestamp +last_edit_timestamp: true +last_edit_time_format: "%b %e %Y at %I:%M %p" + +# Footer "Edit this page on GitHub" link text +gh_edit_link: true +gh_edit_link_text: "Edit this page on GitHub" +gh_edit_repository: "https://github.com/ostreedev/ostree" +gh_edit_branch: "master" +gh_edit_view_mode: "tree" + +compress_html: + clippings: all + comments: all + endings: all + startings: [] + blanklines: false + profile: false diff --git a/docs/_sass/color_schemes/coreos.scss b/docs/_sass/color_schemes/coreos.scss new file mode 100644 index 00000000..a4554be8 --- /dev/null +++ b/docs/_sass/color_schemes/coreos.scss @@ -0,0 +1 @@ +$link-color: #53a3da; From 90b1644f1e83004f43db61fd670ddc079240434b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Mon, 28 Sep 2020 17:02:45 +0200 Subject: [PATCH 44/58] docs: Update Index page --- docs/index.md | 155 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 1 deletion(-) mode change 120000 => 100644 docs/index.md diff --git a/docs/index.md b/docs/index.md deleted file mode 120000 index 32d46ee8..00000000 --- a/docs/index.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..083a69b6 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,154 @@ +--- +nav_order: 1 +--- + +# libostree +{: .no_toc } + +1. TOC +{:toc} + +This project is now known as "libostree", though it is still appropriate to use +the previous name: "OSTree" (or "ostree"). The focus is on projects which use +libostree's shared library, rather than users directly invoking the command line +tools (except for build systems). However, in most of the rest of the +documentation, we will use the term "OSTree", since it's slightly shorter, and +changing all documentation at once is impractical. We expect to transition to +the new name over time. + +As implied above, libostree is both a shared library and suite of command line +tools that combines a "git-like" model for committing and downloading bootable +filesystem trees, along with a layer for deploying them and managing the +bootloader configuration. + +The core OSTree model is like git in that it checksums individual files and has +a content-addressed-object store. It's unlike git in that it "checks out" the +files via hardlinks, and they thus need to be immutable to prevent corruption. +Therefore, another way to think of OSTree is that it's just a more polished +version of +[Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285). + +**Features:** + + - Transactional upgrades and rollback for the system + - Replicating content incrementally over HTTP via GPG signatures and "pinned TLS" support + - Support for parallel installing more than just 2 bootable roots + - Binary history on the server side (and client) + - Introspectable shared library API for build and deployment systems + - Flexible support for multiple branches and repositories, supporting + projects like [flatpak](https://github.com/flatpak/flatpak) which + use libostree for applications, rather than hosts. + +## Operating systems and distributions using OSTree + +[Endless OS](https://endlessos.com/) uses libostree for their host system as +well as flatpak. See +their [eos-updater](https://github.com/endlessm/eos-updater) +and [deb-ostree-builder](https://github.com/dbnicholson/deb-ostree-builder) +projects. + +Fedora derivatives use rpm-ostree (noted below); there are 3 variants using OSTree: + + - [Fedora CoreOS](https://getfedora.org/en/coreos/) + - [Fedora Silverblue](https://silverblue.fedoraproject.org/) + - [Fedora IoT](https://iot.fedoraproject.org/) + +Red Hat Enterprise Linux CoreOS is a derivative of Fedora CoreOS, used in [OpenShift 4](https://try.openshift.com/). +The [machine-config-operator](https://github.com/openshift/machine-config-operator/blob/master/docs/OSUpgrades.md) +manages upgrades. RHEL CoreOS is also the successor to RHEL Atomic Host, which +uses rpm-ostree as well. + +[GNOME Continuous](https://wiki.gnome.org/Projects/GnomeContinuous) is +where OSTree was born - as a high performance continuous delivery/testing +system for GNOME. + +[Liri OS](https://liri.io/download/silverblue/) has the option to install +their distribution using ostree. + +## Distribution build tools + +[meta-updater](https://github.com/advancedtelematic/meta-updater) is +a layer available for [OpenEmbedded](http://www.openembedded.org/wiki/Main_Page) +systems. + +[QtOTA](http://doc.qt.io/QtOTA/) is Qt's over-the-air update framework +which uses libostree. + +The [BuildStream](https://gitlab.com/BuildStream/buildstream) build and +integration tool supports importing and exporting from libostree repos. + +Fedora [coreos-assembler](https://github.com/coreos/coreos-assembler) is +the build tool used to generate Fedora CoreOS derivatives. + +## Projects linking to libostree + +[rpm-ostree](https://github.com/projectatomic/rpm-ostree) is used by the +Fedora-derived operating systems listed above. It is a full hybrid +image/package system. By default it uses libostree to atomically replicate a base OS +(all dependency resolution is done on the server), but it supports "package layering", where +additional RPMs can be layered on top of the base. This brings a "best of both worlds"" +model for image and package systems. + +[eos-updater](https://github.com/endlessm/eos-updater) is a daemon that implements updates +on EndlessOS. + +[flatpak](https://github.com/flatpak/flatpak) uses libostree for desktop +application containers. Unlike most of the other systems here, flatpak does not +use the "libostree host system" aspects (e.g. bootloader management), just the +"git-like hardlink dedup". For example, flatpak supports a per-user OSTree +repository. + +## Language bindings + +libostree is accessible via [GObject Introspection](https://gi.readthedocs.io/en/latest/); +any language which has implemented the GI binding model should work. +For example, Both [pygobject](https://pygobject.readthedocs.io/en/latest/) +and [gjs](https://gitlab.gnome.org/GNOME/gjs) are known to work +and further are actually used in libostree's test suite today. + +Some bindings take the approach of using GI as a lower level and +write higher level manual bindings on top; this is more common +for statically compiled languages. Here's a list of such bindings: + + - [ostree-go](https://github.com/ostreedev/ostree-go/) + - [ostree-rs](https://gitlab.com/fkrull/ostree-rs/) + +## Building + +Releases are available as GPG signed git tags, and most recent +versions support extended validation using +[git-evtag](https://github.com/cgwalters/git-evtag). + +However, in order to build from a git clone, you must update the +submodules. If you're packaging OSTree and want a tarball, I +recommend using a "recursive git archive" script. There are several +available online; +[this code](https://github.com/ostreedev/ostree/blob/master/packaging/Makefile.dist-packaging#L11) +in OSTree is an example. + +Once you have a git clone or recursive archive, building is the +same as almost every autotools project: + +``` +git submodule update --init +env NOCONFIGURE=1 ./autogen.sh +./configure --prefix=... +make +make install DESTDIR=/path/to/dest +``` + +## Contributing + +See [Contributing](docs/CONTRIBUTING.md). + +## Licensing + +The licensing for the *code* of libostree can be canonically found in the individual files; +and the overall status in the [COPYING](https://github.com/ostreedev/ostree/blob/master/COPYING) +file in the source. Currently, that's LGPLv2+. This also covers the man pages and API docs. + +The license for the manual documentation in the `doc/` directory is: +`SPDX-License-Identifier: (CC-BY-SA-3.0 OR GFDL-1.3-or-later)` +This is intended to allow use by Wikipedia and other projects. + +In general, files should have a `SPDX-License-Identifier` and that is canonical. From 558720e7aa1870cbbdb4a0dc22a3968d116daec3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 16 Sep 2020 00:35:33 +0000 Subject: [PATCH 45/58] checkout: Don't hardlink zero sized files Alternative to https://github.com/ostreedev/ostree/pull/2197 Python's (usually) zero-sized `__init__.py` files can provoke us hitting the hardlink limits on some filesystems (`EMLINK`). At least one Fedora rpm-ostree user hit this. The benefits of hardlinking here are quite marginal; lots of hardlinks can behave suboptimally in particular filesystems like BTRFS too. This builds on prior code which made this an option, introduced in https://github.com/ostreedev/ostree/commit/673cacd633f9d6b653cdea530657d3e780a41bbd Now we just do it uncondtionally. Also this provoked a different bug in a very obscure user mode checkout case; when the "real" permissions were different from the "physical" permissions, we would still hardlink. Fix the test case for this. --- man/ostree-checkout.xml | 2 +- src/libostree/ostree-repo-checkout.c | 6 +++++- src/ostree/ot-builtin-checkout.c | 2 +- tests/basic-test.sh | 12 +++++++----- tests/test-libarchive.sh | 18 +++++++++--------- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml index 3956e34f..dfa2ce16 100644 --- a/man/ostree-checkout.xml +++ b/man/ostree-checkout.xml @@ -163,7 +163,7 @@ Boston, MA 02111-1307, USA. - Do not hardlink zero-sized files. + This option does nothing; the functionality is now always on by default. diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 10535a35..00c6a773 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -637,8 +637,12 @@ checkout_one_file_at (OstreeRepo *repo, need_copy = FALSE; } - else if ((options->force_copy_zerosized && is_reg_zerosized) || override_user_unreadable) + else if (is_reg_zerosized || override_user_unreadable) { + /* In https://github.com/ostreedev/ostree/commit/673cacd633f9d6b653cdea530657d3e780a41bbd we + * made this an option, but in order to avoid hitting EMLINK, we now force copy zerosized + * files unconditionally. + */ need_copy = TRUE; } else if (!options->force_copy) diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index adb763a9..fe9558c8 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -84,7 +84,7 @@ static GOptionEntry options[] = { { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" }, { "fsync", 0, 0, G_OPTION_ARG_CALLBACK, parse_fsync_cb, "Specify how to invoke fsync()", "POLICY" }, { "require-hardlinks", 'H', 0, G_OPTION_ARG_NONE, &opt_require_hardlinks, "Do not fall back to full copies if hardlinking fails", NULL }, - { "force-copy-zerosized", 'z', 0, G_OPTION_ARG_NONE, &opt_force_copy_zerosized, "Do not hardlink zero-sized files", NULL }, + { "force-copy-zerosized", 'z', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_force_copy_zerosized, "Do not hardlink zero-sized files", NULL }, { "force-copy", 'C', 0, G_OPTION_ARG_NONE, &opt_force_copy, "Never hardlink (but may reflink if available)", NULL }, { "bareuseronly-dirs", 'M', 0, G_OPTION_ARG_NONE, &opt_bareuseronly_dirs, "Suppress mode bits outside of 0775 for directories (suid, world writable, etc.)", NULL }, { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "FILE" }, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index fc193f4f..9227b0cd 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -750,15 +750,17 @@ rm files -rf && mkdir files touch files/anemptyfile touch files/anotheremptyfile $CMD_PREFIX ostree --repo=repo commit --consume -b tree-with-empty-files --tree=dir=files +$CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} tree-with-empty-files tree-with-empty-files +if files_are_hardlinked tree-with-empty-files/an{,other}emptyfile; then + fatal "--force-copy-zerosized failed" +fi +# And pass the now-defunct -z option to validate it does nothing +rm tree-with-empty-files -rf $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} -z tree-with-empty-files tree-with-empty-files if files_are_hardlinked tree-with-empty-files/an{,other}emptyfile; then fatal "--force-copy-zerosized failed" fi -rm tree-with-empty-files -rf -$CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} tree-with-empty-files tree-with-empty-files -assert_files_hardlinked tree-with-empty-files/an{,other}emptyfile -rm tree-with-empty-files -rf -echo "ok checkout --force-copy-zerosized" +echo "ok checkout zero sized files are not hardlinked" # These should merge, they're identical $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} --union-identical -z tree-with-empty-files tree-with-empty-files diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index 174be800..73a58ddd 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -37,7 +37,7 @@ mkdir foo cd foo mkdir -p usr/bin usr/lib echo contents > usr/bin/foo -touch usr/bin/foo0 +echo foo0 > usr/bin/foo0 ln usr/bin/foo usr/bin/bar ln usr/bin/foo0 usr/bin/bar0 ln -s foo usr/bin/sl @@ -45,8 +45,8 @@ mkdir -p usr/local/bin ln usr/bin/foo usr/local/bin/baz ln usr/bin/foo0 usr/local/bin/baz0 ln usr/bin/sl usr/local/bin/slhl -touch usr/bin/setuidme -touch usr/bin/skipme +echo setuidme > usr/bin/setuidme +echo skipme > usr/bin/skipme echo "a library" > usr/lib/libfoo.so echo "another library" > usr/lib/libbar.so @@ -102,9 +102,9 @@ assert_valid_content () { assert_file_has_content usr/bin/foo contents assert_file_has_content usr/bin/bar contents assert_file_has_content usr/local/bin/baz contents - assert_file_empty usr/bin/foo0 - assert_file_empty usr/bin/bar0 - assert_file_empty usr/local/bin/baz0 + assert_file_has_content usr/bin/foo0 foo0 + assert_file_has_content usr/bin/bar0 foo0 + assert_file_has_content usr/local/bin/baz0 foo0 assert_file_has_content usr/lib/libfoo.so 'a library' assert_file_has_content usr/lib/libbar.so 'another library' @@ -244,7 +244,7 @@ ${CMD_PREFIX} ostree --repo=repo2 commit \ --generate-sizes \ --tree=tar=foo.tar.gz ${CMD_PREFIX} ostree --repo=repo2 show --print-sizes test-tar > sizes.txt -assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.1[  ]kB' -assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/900[  ]bytes' -assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/12' +assert_file_has_content sizes.txt 'Compressed size (needed/total): 0[  ]bytes/1.2[  ]kB' +assert_file_has_content sizes.txt 'Unpacked size (needed/total): 0[  ]bytes/921[  ]bytes' +assert_file_has_content sizes.txt 'Number of objects (needed/total): 0/14' echo "ok tar sizes metadata" From 6ca312a92399c92b60d83eae161a8b8ea148989b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Wed, 30 Sep 2020 19:22:29 +0200 Subject: [PATCH 46/58] docs: Update Contributing and tutorial pages --- docs/CONTRIBUTING.md | 25 +++++++++++++++---------- docs/contributing-tutorial.md | 21 +++++++-------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index de14c380..912ea4a8 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,5 +1,14 @@ -Submitting patches ------------------- +--- +nav_order: 19 +--- + +# Contributing +{: .no_toc } + +1. TOC +{:toc} + +## Submitting patches A majority of current maintainers prefer the GitHub pull request model, and this motivated moving the primary git repository to @@ -26,8 +35,7 @@ Alternative methods if you don't like GitHub (also fully supported): It is likely however once a patch is ready to apply a maintainer will push it to a GitHub PR, and merge via Homu. -Commit message style --------------------- +## Commit message style Please look at `git log` and match the commit log style, which is very similar to the @@ -66,15 +74,13 @@ For more information see [How to Write a Git Commit Message](https://chris.beams To edit the message from the most recent commit run `git commit --amend`. To change older commits on the branch use `git rebase -i`. For a successful rebase have the branch track `upstream master`. Once the changes have been made and saved, run `git push --force origin `. -Running the test suite ----------------------- +## Running the test suite OSTree uses both `make check` and supports the [Installed Tests](https://wiki.gnome.org/GnomeGoals/InstalledTests) model as well (if `--enable-installed-tests` is provided). -Coding style ------------- +## Coding style Indentation is GNU. Files should start with the appropriate mode lines. @@ -168,7 +174,6 @@ Instead do this: } } -Contributing Tutorial ---------------------- +## Contributing Tutorial For a detailed walk-through on building, modifying, and testing, see this [tutorial on how to start contributing to OSTree](contributing-tutorial.md). diff --git a/docs/contributing-tutorial.md b/docs/contributing-tutorial.md index 47d0a1e9..bb656f0e 100644 --- a/docs/contributing-tutorial.md +++ b/docs/contributing-tutorial.md @@ -1,21 +1,14 @@ +--- +nav_order: 20 +--- + # OSTree Contributing Tutorial +{: .no_toc } The following guide is about OSTree forking, building, adding a command, testing the command, and submitting the change. -- [Getting Started](#getting-started) -- [Building OSTree](#building-ostree) - - [Install Build Dependencies](#install-build-dependencies) - - [OSTree Build Commands](#ostree-build-commands) -- [Testing a Build](#testing-a-build) - - [Testing in a Container](#testing-in-a-container) - - [Testing in a Virtual Machine](#testing-in-a-virtual-machine) -- [Tutorial: Adding a basic builtin command to OSTree](#tutorial-adding-a-basic-builtin-command-to-ostree) - - [Modifying OSTree](#modifying-ostree) - - [OSTree Tests](#ostree-tests) - - [Submitting a Patch](#submitting-a-patch) - - [Returning Workflow](#returning-workflow) - ---- +1. TOC +{:toc} ## Getting Started From 68ac9e9c50d0e190a5a82bef5fdea29bd46fdd0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Fri, 2 Oct 2020 14:34:33 +0200 Subject: [PATCH 47/58] docs: Move and update pages from the manual --- docs/{manual => }/adapting-existing.md | 8 ++++++++ docs/{manual => }/atomic-upgrades.md | 8 ++++++++ docs/{manual => }/buildsystem-and-repos.md | 8 ++++++++ docs/{manual => }/deployment.md | 8 ++++++++ docs/{manual => }/formats.md | 8 ++++++++ docs/{manual => }/introduction.md | 8 ++++++++ docs/{manual => }/related-projects.md | 8 ++++++++ docs/{manual => }/repo.md | 8 ++++++++ docs/{manual => }/repository-management.md | 8 ++++++++ 9 files changed, 72 insertions(+) rename docs/{manual => }/adapting-existing.md (99%) rename docs/{manual => }/atomic-upgrades.md (99%) rename docs/{manual => }/buildsystem-and-repos.md (99%) rename docs/{manual => }/deployment.md (99%) rename docs/{manual => }/formats.md (99%) rename docs/{manual => }/introduction.md (99%) rename docs/{manual => }/related-projects.md (99%) rename docs/{manual => }/repo.md (99%) rename docs/{manual => }/repository-management.md (99%) diff --git a/docs/manual/adapting-existing.md b/docs/adapting-existing.md similarity index 99% rename from docs/manual/adapting-existing.md rename to docs/adapting-existing.md index 3a1b8d69..cc4b76d2 100644 --- a/docs/manual/adapting-existing.md +++ b/docs/adapting-existing.md @@ -1,4 +1,12 @@ +--- +nav_order: 6 +--- + # Adapting existing mainstream distributions +{: .no_toc } + +1. TOC +{:toc} ## System layout diff --git a/docs/manual/atomic-upgrades.md b/docs/atomic-upgrades.md similarity index 99% rename from docs/manual/atomic-upgrades.md rename to docs/atomic-upgrades.md index f2c01cc1..3ddd8b40 100644 --- a/docs/manual/atomic-upgrades.md +++ b/docs/atomic-upgrades.md @@ -1,4 +1,12 @@ +--- +nav_order: 5 +--- + # Atomic Upgrades +{: .no_toc } + +1. TOC +{:toc} ## You can turn off the power anytime you want... diff --git a/docs/manual/buildsystem-and-repos.md b/docs/buildsystem-and-repos.md similarity index 99% rename from docs/manual/buildsystem-and-repos.md rename to docs/buildsystem-and-repos.md index fbae0322..6d506b4e 100644 --- a/docs/manual/buildsystem-and-repos.md +++ b/docs/buildsystem-and-repos.md @@ -1,4 +1,12 @@ +--- +nav_order: 8 +--- + # Writing a buildsystem and managing repositories +{: .no_toc } + +1. TOC +{:toc} OSTree is not a package system. It does not directly support building source code. Rather, it is a tool for transporting and managing diff --git a/docs/manual/deployment.md b/docs/deployment.md similarity index 99% rename from docs/manual/deployment.md rename to docs/deployment.md index afbcbabb..1ea7ea46 100644 --- a/docs/manual/deployment.md +++ b/docs/deployment.md @@ -1,4 +1,12 @@ +--- +nav_order: 4 +--- + # Deployments +{: .no_toc } + +1. TOC +{:toc} ## Overview diff --git a/docs/manual/formats.md b/docs/formats.md similarity index 99% rename from docs/manual/formats.md rename to docs/formats.md index 884b1b5e..36d395bd 100644 --- a/docs/manual/formats.md +++ b/docs/formats.md @@ -1,4 +1,12 @@ +--- +nav_order: 7 +--- + # OSTree data formats +{: .no_toc } + +1. TOC +{:toc} ## On the topic of "smart servers" diff --git a/docs/manual/introduction.md b/docs/introduction.md similarity index 99% rename from docs/manual/introduction.md rename to docs/introduction.md index c0113f5d..a6fa2252 100644 --- a/docs/manual/introduction.md +++ b/docs/introduction.md @@ -1,4 +1,12 @@ +--- +nav_order: 2 +--- + # OSTree Overview +{: .no_toc } + +1. TOC +{:toc} ## Introduction diff --git a/docs/manual/related-projects.md b/docs/related-projects.md similarity index 99% rename from docs/manual/related-projects.md rename to docs/related-projects.md index 429b46b9..7ddf043f 100644 --- a/docs/manual/related-projects.md +++ b/docs/related-projects.md @@ -1,4 +1,12 @@ +--- +nav_order: 10 +--- + # Related Projects +{: .no_toc } + +1. TOC +{:toc} OSTree is in many ways very evolutionary. It builds on concepts and ideas introduced from many different projects such as diff --git a/docs/manual/repo.md b/docs/repo.md similarity index 99% rename from docs/manual/repo.md rename to docs/repo.md index 8746163e..5cc59bf1 100644 --- a/docs/manual/repo.md +++ b/docs/repo.md @@ -1,4 +1,12 @@ +--- +nav_order: 3 +--- + # Anatomy of an OSTree repository +{: .no_toc } + +1. TOC +{:toc} ## Core object types and data model diff --git a/docs/manual/repository-management.md b/docs/repository-management.md similarity index 99% rename from docs/manual/repository-management.md rename to docs/repository-management.md index 77519fb9..11fe2f40 100644 --- a/docs/manual/repository-management.md +++ b/docs/repository-management.md @@ -1,4 +1,12 @@ +--- +nav_order: 9 +--- + # Managing content in OSTree repositories +{: .no_toc } + +1. TOC +{:toc} Once you have a build system going, if you actually want client systems to retrieve the content, you will quickly feel a need for From 13844c6b3e8e60fdab1c207a350d6dc274450bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Fri, 2 Oct 2020 14:38:20 +0200 Subject: [PATCH 48/58] docs: Move historical README to the docs --- README-historical.md => docs/README-historical.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) rename README-historical.md => docs/README-historical.md (98%) diff --git a/README-historical.md b/docs/README-historical.md similarity index 98% rename from README-historical.md rename to docs/README-historical.md index 732f8bdc..eebf5b2f 100644 --- a/README-historical.md +++ b/docs/README-historical.md @@ -1,6 +1,11 @@ -This file is outdated, but some of the text here is still useful for +--- +nav_order: 99 +title: Historical OSTree README +--- + +**This file is outdated, but some of the text here is still useful for historical context. I'm preserving it (explicitly still in the tree) -for posterity. +for posterity.** OSTree ====== @@ -49,7 +54,7 @@ Comparison with existing tools Now your system is in an undefined state. You can use e.g. rpm -qV to try to find out what you overwrote, but neither dpkg nor rpm will help clean up any files left over that aren't shipped by - the old package. + the old package. This is most realistic option for people hacking on system components currently, but ostree will be better. @@ -205,7 +210,7 @@ handling of binaries is very generic and unoptimized. In contrast, ostree is explicitly designed for binaries, and in particular one type of binary - ELF executables (or it will be once we -start using bsdiff). +start using bsdiff). Another big difference versus git is that ostree uses hard links between "checkouts" and the repository. This means each checkout uses From 797eb3b14f840c960c218c6abfa68dc19e2839ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Fri, 2 Oct 2020 14:40:16 +0200 Subject: [PATCH 49/58] README: Update and mention new docs --- README.md | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1ef8e302..be6f6453 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,4 @@ -libostree ---------- - -New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) - ------ +# libostree This project is now known as "libostree", though it is still appropriate to use the previous name: "OSTree" (or "ostree"). The focus is on projects which use @@ -36,8 +31,12 @@ version of projects like [flatpak](https://github.com/flatpak/flatpak) which use libostree for applications, rather than hosts. -Operating systems and distributions using OSTree ---------------------- +## Documentation + +For more information, see the [project documentation](docs/index.md) or the +[project documentation website](https://ostreedev.github.io/ostree). + +## Operating systems and distributions using OSTree [Endless OS](https://endlessos.com/) uses libostree for their host system as well as flatpak. See @@ -63,8 +62,7 @@ system for GNOME. [Liri OS](https://liri.io/download/silverblue/) has the option to install their distribution using ostree. -Distribution build tools ------------------------- +## Distribution build tools [meta-updater](https://github.com/advancedtelematic/meta-updater) is a layer available for [OpenEmbedded](http://www.openembedded.org/wiki/Main_Page) @@ -79,8 +77,7 @@ integration tool supports importing and exporting from libostree repos. Fedora [coreos-assembler](https://github.com/coreos/coreos-assembler) is the build tool used to generate Fedora CoreOS derivatives. -Projects linking to libostree ------------------------------ +## Projects linking to libostree [rpm-ostree](https://github.com/projectatomic/rpm-ostree) is used by the Fedora-derived operating systems listed above. It is a full hybrid @@ -98,8 +95,7 @@ use the "libostree host system" aspects (e.g. bootloader management), just the "git-like hardlink dedup". For example, flatpak supports a per-user OSTree repository. -Language bindings ----- +## Language bindings libostree is accessible via [GObject Introspection](https://gi.readthedocs.io/en/latest/); any language which has implemented the GI binding model should work. @@ -114,8 +110,7 @@ for statically compiled languages. Here's a list of such bindings: - [ostree-go](https://github.com/ostreedev/ostree-go/) - [ostree-rs](https://gitlab.com/fkrull/ostree-rs/) -Building --------- +## Building Releases are available as GPG signed git tags, and most recent versions support extended validation using @@ -139,19 +134,11 @@ make make install DESTDIR=/path/to/dest ``` -More documentation ------------------- - -New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) - -Contributing ------------- +## Contributing See [Contributing](docs/CONTRIBUTING.md). - -Licensing -------- +## Licensing The licensing for the *code* of libostree can be canonically found in the individual files; and the overall status in the [COPYING](https://github.com/ostreedev/ostree/blob/master/COPYING) From 3f6eb2264e30dc0b0f74b3e31569f29085a14a80 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Oct 2020 14:04:21 -0400 Subject: [PATCH 50/58] libglnx: Bump to master To pull in the fix for `/var/tmp` on NixOS but also on general principle. Update submodule: libglnx ``` Colin Walters (1): xattrs: Add better error prefixing Rebecca Turner (2): glnx-fdio: try $TMPDIR if /var/tmp doesn't exist glnx-fdio: use $TMPDIR if set ``` --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 84b981a2..1dd01d5e 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 84b981a2ef65b46040f7529839448d9219687ac8 +Subproject commit 1dd01d5ef172fbe7cb385c91ee2a3740962e8074 From 0bbd89e32652652ed080f7a1263697ab04b9d2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Ravier?= Date: Mon, 5 Oct 2020 21:10:31 +0200 Subject: [PATCH 51/58] docs: Fix URL in Jekyll _config.yml --- docs/_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_config.yml b/docs/_config.yml index 2070f1cf..835733a0 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,7 +1,7 @@ title: ostreedev/ostree description: ostree documentation baseurl: "/ostree" -url: "https://coreos.github.io" +url: "https://ostreedev.github.io" # Comment above and use below for local development # url: "http://localhost:4000" permalink: /:title/ From 9c51aa309003cdad60915622cdec9e2068072a43 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Sat, 3 Oct 2020 11:23:49 -0400 Subject: [PATCH 52/58] Add Packit integration This is a basic `.packit.yaml` integration file which will allow us have continuous builds of OSTree in cosa and upstream CI. If things go well, we'll likely deploy this in other build tools like rpm-ostree. Prompted by wanting to get #2155 out to unblock https://github.com/coreos/rpm-ostree/pull/2170. --- .packit.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .packit.yaml diff --git a/.packit.yaml b/.packit.yaml new file mode 100644 index 00000000..0aec81ed --- /dev/null +++ b/.packit.yaml @@ -0,0 +1,11 @@ +# build into f32-coreos-continuous on every commit to master +jobs: + - job: production_build + trigger: commit + metadata: + branch: master + targets: f32-coreos-continuous +specfile_path: ostree.spec +actions: + # https://packit.dev/faq/#how-can-i-download-rpm-spec-file-if-it-is-not-part-of-upstream-repository + post-upstream-clone: "wget https://src.fedoraproject.org/rpms/ostree/raw/master/f/ostree.spec" From b3dc074f5ed6d3a1e40dbc1e3315a518512d3c63 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 7 Oct 2020 14:44:25 -0400 Subject: [PATCH 53/58] lib/deploy: Don't leak fd when checksumming dtbs Likely the root of https://bugzilla.redhat.com/show_bug.cgi?id=1886149. --- src/libostree/ostree-sysroot-deploy.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 1c4fb5dc..7b7ba5e9 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -272,13 +272,13 @@ checksum_dir_recurse (int dfd, } else { - int fd; + glnx_autofd int fd = -1; if (!ot_openat_ignore_enoent (dfditer.fd, d_name, &fd, error)) return FALSE; if (fd != -1) { - g_autoptr(GInputStream) in = g_unix_input_stream_new (fd, FALSE); + g_autoptr(GInputStream) in = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error)) return FALSE; } From ee632e4968e9f5aade3f0ace8ec90a84a45259b9 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 7 Oct 2020 15:04:17 -0400 Subject: [PATCH 54/58] ci: Make Packit ignore downstream patches We don't really carry "Fedora-only" patches in dist-git. So we want to nuke all the patches which exist there. Follow-up to #2210. --- .packit.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.packit.yaml b/.packit.yaml index 0aec81ed..ebfe5648 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -8,4 +8,7 @@ jobs: specfile_path: ostree.spec actions: # https://packit.dev/faq/#how-can-i-download-rpm-spec-file-if-it-is-not-part-of-upstream-repository - post-upstream-clone: "wget https://src.fedoraproject.org/rpms/ostree/raw/master/f/ostree.spec" + post-upstream-clone: + - "wget https://src.fedoraproject.org/rpms/ostree/raw/master/f/ostree.spec" + # we don't want any downstream patches + - "sed -ie 's/^Patch/# Patch/g' ostree.spec" From 6ed1066ef08e3f56b6c09031e82343cc8e0a99d1 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 14 Sep 2020 13:52:10 +0200 Subject: [PATCH 55/58] ostree_repo_find_remotes_async: Fix leak of summary We were creating a GVariant from a GBytes and storing it in an g_autoptr without ref_sinking it. --- src/libostree/ostree-repo-pull.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f16ccec5..ab32b54e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -5372,8 +5372,8 @@ find_remotes_cb (GObject *obj, /* Check the metadata in the summary file, especially whether it contains * all the @refs we are interested in. */ - summary_v = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, - summary_bytes, FALSE); + summary_v = g_variant_ref_sink (g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, + summary_bytes, FALSE)); /* Check the summary’s additional metadata and set up @commit_metadata * and @refs_and_remotes_table with the refs listed in the summary file, From f821cdb89ec591d5195c4093ba6a9da43a703463 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 16 Sep 2020 12:09:41 +0200 Subject: [PATCH 56/58] fetch_summary_with_options: Fix n-network-retries option parsing "&u" is not a valid gvariant format string, it should just be "u". --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ab32b54e..58c80543 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -6127,7 +6127,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-url", "&s", &url_override); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent); - (void) g_variant_lookup (options, "n-network-retries", "&u", &n_network_retries); + (void) g_variant_lookup (options, "n-network-retries", "u", &n_network_retries); } if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) From 2e9db809b9675585296ef410d5af4d84f9074101 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 16 Sep 2020 15:54:03 +0200 Subject: [PATCH 57/58] signatures: Fix leak in _sign_detached_metadata_append() This needs to ref_sink the returned variant, as it is used with g_autoptr in the callers. --- src/libostree/ostree-sign.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index ee7e928d..eeef96dd 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -365,7 +365,7 @@ _sign_detached_metadata_append (OstreeSign *self, signature_key, g_variant_builder_end (signature_builder)); - return g_variant_dict_end (&metadata_dict); + return g_variant_ref_sink (g_variant_dict_end (&metadata_dict)); } /** From 32a3a1297312e566df3141c6c7e3b99709e474b1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 13 Oct 2020 14:31:26 -0400 Subject: [PATCH 58/58] Release 2020.7 --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 14 -------------- src/libostree/libostree-released.sym | 14 ++++++++++++++ tests/test-symbols.sh | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/configure.ac b/configure.ac index de219eab..13f5bef8 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ m4_define([year_version], [2020]) m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 341a22a8..2ed658c4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,20 +17,6 @@ Boston, MA 02111-1307, USA. ***/ -LIBOSTREE_2020.7 { -global: - /* Add symbols here, and uncomment the bits in - * Makefile-libostree.am to enable this too. - */ - ostree_repo_static_delta_execute_offline_with_signature; - ostree_repo_static_delta_verify_signature; - ostree_bootconfig_parser_get_overlay_initrds; - ostree_bootconfig_parser_set_overlay_initrds; - ostree_sysroot_deploy_tree_with_options; - ostree_sysroot_stage_tree_with_options; - ostree_sysroot_stage_overlay_initrd; -} LIBOSTREE_2020.4; - /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index e7d985ca..c154d8c1 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -622,6 +622,20 @@ global: /* No new symbols in 2020.6 */ +LIBOSTREE_2020.7 { +global: + /* Add symbols here, and uncomment the bits in + * Makefile-libostree.am to enable this too. + */ + ostree_repo_static_delta_execute_offline_with_signature; + ostree_repo_static_delta_verify_signature; + ostree_bootconfig_parser_get_overlay_initrds; + ostree_bootconfig_parser_set_overlay_initrds; + ostree_sysroot_deploy_tree_with_options; + ostree_sysroot_stage_tree_with_options; + ostree_sysroot_stage_overlay_initrd; +} LIBOSTREE_2020.4; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 697d9ad1..e4a0a943 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -66,7 +66,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <