From 3f852d944378a9997f769a3da7dc0ec253a4490f Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 12 Jul 2021 08:40:14 +0000 Subject: [PATCH 01/47] configure: post-release version bump --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index c3f700ad..4002a8c7 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.63]) dnl To perform a release, follow the instructions in `docs/CONTRIBUTING.md`. m4_define([year_version], [2021]) -m4_define([release_version], [3]) +m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 5b5d3b12fb3f6ef4875f2fd2b65e99a8b3fdfb91 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Wed, 14 Jul 2021 16:07:16 -0400 Subject: [PATCH 02/47] man: improve statoverride description A statoverride file written in the obvious way will produce incorrect results for two independent reasons. Document them. --- man/ostree-commit.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index 81af7bf2..e1e3a8b9 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -232,7 +232,9 @@ Boston, MA 02111-1307, USA. ="PATH" - File containing list of modifications to make permissions (file mode, followed by space, followed by file path). + File containing list of modifications to make permissions (file mode in + decimal, followed by space, followed by file path). The specified mode + is ORed with the file's original mode unless preceded by "=". From dba2cdcbac5c064574f44a56714324e71ed9500b Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 26 Jul 2019 09:38:23 -0600 Subject: [PATCH 03/47] lib/repo: Factor out GPG verifier key imports Currently the verifier only imports all the GPG keys when verifying data, but it would also be useful for inspecting the trusted keys. --- src/libostree/ostree-gpg-verifier.c | 173 +++++++++++++++------------- 1 file changed, 96 insertions(+), 77 deletions(-) diff --git a/src/libostree/ostree-gpg-verifier.c b/src/libostree/ostree-gpg-verifier.c index 95ed36ee..88850d46 100644 --- a/src/libostree/ostree-gpg-verifier.c +++ b/src/libostree/ostree-gpg-verifier.c @@ -91,6 +91,100 @@ verify_result_finalized_cb (gpointer data, (void) glnx_shutil_rm_rf_at (AT_FDCWD, tmp_dir, NULL, NULL); } +static gboolean +_ostree_gpg_verifier_import_keys (OstreeGpgVerifier *self, + gpgme_ctx_t gpgme_ctx, + GOutputStream *pubring_stream, + GCancellable *cancellable, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR("GPG", error); + + for (GList *link = self->keyrings; link != NULL; link = link->next) + { + g_autoptr(GFileInputStream) source_stream = NULL; + GFile *keyring_file = link->data; + gssize bytes_written; + GError *local_error = NULL; + + source_stream = g_file_read (keyring_file, cancellable, &local_error); + + /* Disregard non-existent keyrings. */ + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + { + g_clear_error (&local_error); + continue; + } + else if (local_error != NULL) + { + g_propagate_error (error, local_error); + return FALSE; + } + + bytes_written = g_output_stream_splice (pubring_stream, + G_INPUT_STREAM (source_stream), + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, + cancellable, error); + if (bytes_written < 0) + return FALSE; + } + + for (guint i = 0; i < self->keyring_data->len; i++) + { + GBytes *keyringd = self->keyring_data->pdata[i]; + gsize len; + gsize bytes_written; + const guint8 *buf = g_bytes_get_data (keyringd, &len); + if (!g_output_stream_write_all (pubring_stream, buf, len, &bytes_written, + cancellable, error)) + return FALSE; + } + + if (!g_output_stream_close (pubring_stream, cancellable, error)) + return FALSE; + + /* Save the previous armor value - we need it on for importing ASCII keys */ + gboolean ret = FALSE; + int armor = gpgme_get_armor (gpgme_ctx); + gpgme_set_armor (gpgme_ctx, 1); + + /* Now, use the API to import ASCII-armored keys */ + if (self->key_ascii_files) + { + for (guint i = 0; i < self->key_ascii_files->len; i++) + { + gpgme_error_t gpg_error; + const char *path = self->key_ascii_files->pdata[i]; + glnx_autofd int fd = -1; + g_auto(gpgme_data_t) kdata = NULL; + + if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error)) + goto out; + + gpg_error = gpgme_data_new_from_fd (&kdata, fd); + if (gpg_error != GPG_ERR_NO_ERROR) + { + ot_gpgme_throw (gpg_error, error, "Loading data from fd %i", fd); + goto out; + } + + gpg_error = gpgme_op_import (gpgme_ctx, kdata); + if (gpg_error != GPG_ERR_NO_ERROR) + { + ot_gpgme_throw (gpg_error, error, "Failed to import key"); + goto out; + } + } + } + + ret = TRUE; + + out: + gpgme_set_armor (gpgme_ctx, armor); + + return ret; +} + OstreeGpgVerifyResult * _ostree_gpg_verifier_check_signature (OstreeGpgVerifier *self, GBytes *signed_data, @@ -106,8 +200,6 @@ _ostree_gpg_verifier_check_signature (OstreeGpgVerifier *self, g_autoptr(GOutputStream) target_stream = NULL; OstreeGpgVerifyResult *result = NULL; gboolean success = FALSE; - GList *link; - int armor; /* GPGME has no API for using multiple keyrings (aka, gpg --keyring), * so we concatenate all the keyring files into one pubring.gpg in a @@ -127,83 +219,10 @@ _ostree_gpg_verifier_check_signature (OstreeGpgVerifier *self, cancellable, error)) goto out; - for (link = self->keyrings; link != NULL; link = link->next) - { - g_autoptr(GFileInputStream) source_stream = NULL; - GFile *keyring_file = link->data; - gssize bytes_written; - GError *local_error = NULL; - - source_stream = g_file_read (keyring_file, cancellable, &local_error); - - /* Disregard non-existent keyrings. */ - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&local_error); - continue; - } - else if (local_error != NULL) - { - g_propagate_error (error, local_error); - goto out; - } - - bytes_written = g_output_stream_splice (target_stream, - G_INPUT_STREAM (source_stream), - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, - cancellable, error); - if (bytes_written < 0) - goto out; - } - - for (guint i = 0; i < self->keyring_data->len; i++) - { - GBytes *keyringd = self->keyring_data->pdata[i]; - gsize len; - gsize bytes_written; - const guint8 *buf = g_bytes_get_data (keyringd, &len); - if (!g_output_stream_write_all (target_stream, buf, len, &bytes_written, - cancellable, error)) - goto out; - } - - if (!g_output_stream_close (target_stream, cancellable, error)) + if (!_ostree_gpg_verifier_import_keys (self, result->context, target_stream, + cancellable, error)) goto out; - /* Save the previous armor value - we need it on for importing ASCII keys */ - armor = gpgme_get_armor (result->context); - gpgme_set_armor (result->context, 1); - - /* Now, use the API to import ASCII-armored keys */ - if (self->key_ascii_files) - { - for (guint i = 0; i < self->key_ascii_files->len; i++) - { - const char *path = self->key_ascii_files->pdata[i]; - glnx_autofd int fd = -1; - g_auto(gpgme_data_t) kdata = NULL; - - if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error)) - goto out; - - gpg_error = gpgme_data_new_from_fd (&kdata, fd); - if (gpg_error != GPG_ERR_NO_ERROR) - { - ot_gpgme_throw (gpg_error, error, "Loading data from fd %i", fd); - goto out; - } - - gpg_error = gpgme_op_import (result->context, kdata); - if (gpg_error != GPG_ERR_NO_ERROR) - { - ot_gpgme_throw (gpg_error, error, "Failed to import key"); - goto out; - } - } - } - - gpgme_set_armor (result->context, armor); - /* Both the signed data and signature GBytes instances will outlive the * gpgme_data_t structs, so we can safely reuse the GBytes memory buffer * directly and avoid a copy. */ From c8715c123e7979b078d45fd18e7cf5a276eda698 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 13 Aug 2019 10:36:11 -0600 Subject: [PATCH 04/47] lib/repo: Factor out GPG verifier preparation In order to use the GPG verifier, it needs to be seeded with GPG keys after instantation. Currently this is only used for verifying data, but it will also be used for getting a list of trusted GPG keys in a subsequent commit. --- src/libostree/ostree-repo.c | 61 +++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b90e1c13..d7b38374 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5338,28 +5338,25 @@ find_keyring (OstreeRepo *self, return TRUE; } -static OstreeGpgVerifyResult * -_ostree_repo_gpg_verify_data_internal (OstreeRepo *self, - const gchar *remote_name, - GBytes *data, - GBytes *signatures, - GFile *keyringdir, - GFile *extra_keyring, - GCancellable *cancellable, - GError **error) +static gboolean +_ostree_repo_gpg_prepare_verifier (OstreeRepo *self, + const gchar *remote_name, + GFile *keyringdir, + GFile *extra_keyring, + OstreeGpgVerifier **out_verifier, + GCancellable *cancellable, + GError **error) { - g_autoptr(OstreeGpgVerifier) verifier = NULL; + g_autoptr(OstreeGpgVerifier) verifier = _ostree_gpg_verifier_new (); gboolean add_global_keyring_dir = TRUE; - verifier = _ostree_gpg_verifier_new (); - if (remote_name == OSTREE_ALL_REMOTES) { /* Add all available remote keyring files. */ if (!_ostree_gpg_verifier_add_keyring_dir_at (verifier, self->repo_dir_fd, ".", cancellable, error)) - return NULL; + return FALSE; } else if (remote_name != NULL) { @@ -5369,11 +5366,11 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, remote = _ostree_repo_get_remote_inherited (self, remote_name, error); if (remote == NULL) - return NULL; + return FALSE; g_autoptr(GBytes) keyring_data = NULL; if (!find_keyring (self, remote, &keyring_data, cancellable, error)) - return NULL; + return FALSE; if (keyring_data != NULL) { @@ -5389,14 +5386,14 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, ";,", &gpgkeypath_list, error)) - return NULL; + return FALSE; if (gpgkeypath_list) { for (char **iter = gpgkeypath_list; *iter != NULL; ++iter) if (!_ostree_gpg_verifier_add_keyfile_path (verifier, *iter, cancellable, error)) - return NULL; + return FALSE; } } @@ -5404,20 +5401,46 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, { /* Use the deprecated global keyring directory. */ if (!_ostree_gpg_verifier_add_global_keyring_dir (verifier, cancellable, error)) - return NULL; + return FALSE; } if (keyringdir) { if (!_ostree_gpg_verifier_add_keyring_dir (verifier, keyringdir, cancellable, error)) - return NULL; + return FALSE; } if (extra_keyring != NULL) { _ostree_gpg_verifier_add_keyring_file (verifier, extra_keyring); } + if (out_verifier != NULL) + *out_verifier = g_steal_pointer (&verifier); + + return TRUE; +} + +static OstreeGpgVerifyResult * +_ostree_repo_gpg_verify_data_internal (OstreeRepo *self, + const gchar *remote_name, + GBytes *data, + GBytes *signatures, + GFile *keyringdir, + GFile *extra_keyring, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(OstreeGpgVerifier) verifier = NULL; + if (!_ostree_repo_gpg_prepare_verifier (self, + remote_name, + keyringdir, + extra_keyring, + &verifier, + cancellable, + error)) + return NULL; + return _ostree_gpg_verifier_check_signature (verifier, data, signatures, From fc073654dca64dacdc0067bd35ecb82f8c794fe3 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 14 Jul 2021 11:04:59 -0600 Subject: [PATCH 05/47] lib/repo: Allow preparing GPG verifier without global keyrings Currently the verifier decides whether to include the global keyrings based on whether the specified remote has its own keyring or not. Allow callers to exclude the global keyrings even when that's not the case. This will be used in a subsequent commit in order to get the GPG keys only associated with a remote. --- src/libostree/ostree-repo.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d7b38374..254f7010 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5343,12 +5343,12 @@ _ostree_repo_gpg_prepare_verifier (OstreeRepo *self, const gchar *remote_name, GFile *keyringdir, GFile *extra_keyring, + gboolean add_global_keyrings, OstreeGpgVerifier **out_verifier, GCancellable *cancellable, GError **error) { g_autoptr(OstreeGpgVerifier) verifier = _ostree_gpg_verifier_new (); - gboolean add_global_keyring_dir = TRUE; if (remote_name == OSTREE_ALL_REMOTES) { @@ -5375,7 +5375,7 @@ _ostree_repo_gpg_prepare_verifier (OstreeRepo *self, if (keyring_data != NULL) { _ostree_gpg_verifier_add_keyring_data (verifier, keyring_data, remote->keyring); - add_global_keyring_dir = FALSE; + add_global_keyrings = FALSE; } g_auto(GStrv) gpgkeypath_list = NULL; @@ -5397,7 +5397,7 @@ _ostree_repo_gpg_prepare_verifier (OstreeRepo *self, } } - if (add_global_keyring_dir) + if (add_global_keyrings) { /* Use the deprecated global keyring directory. */ if (!_ostree_gpg_verifier_add_global_keyring_dir (verifier, cancellable, error)) @@ -5436,6 +5436,7 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, remote_name, keyringdir, extra_keyring, + TRUE, &verifier, cancellable, error)) From a50f6d0b9fa1d3079ff5bf78e46da3a635a37611 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 13 Aug 2019 13:36:00 -0600 Subject: [PATCH 06/47] lib/repo: Add ostree_repo_remote_get_gpg_keys() This function enumerates the trusted GPG keys for a remote and returns an array of `GVariant`s describing them. This is useful to see which keys are collected by ostree for a particular remote. The same information can be gathered with `gpg`. However, since ostree allows multiple keyring locations, that's only really useful if you have knowledge of how ostree collects GPG keyrings. The format of the variants is documented in `OSTREE_GPG_KEY_GVARIANT_FORMAT`. This format is primarily a copy of selected fields within `gpgme_key_t` and its subtypes. The fields are placed within vardicts rather than using a more efficient tuple of concrete types. This will allow flexibility if more components of `gpgme_key_t` are desired in the future. --- Makefile-libostree.am | 6 +- apidoc/ostree-sections.txt | 3 + src/libostree/libostree-devel.sym | 5 ++ src/libostree/ostree-gpg-verifier.c | 85 +++++++++++++++++++ src/libostree/ostree-gpg-verifier.h | 6 ++ src/libostree/ostree-repo.c | 124 ++++++++++++++++++++++++++++ src/libostree/ostree-repo.h | 40 +++++++++ 7 files changed, 266 insertions(+), 3 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index dd396974..d40de48d 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -173,9 +173,9 @@ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym # Uncomment this include when adding new development symbols. -#if BUILDOPT_IS_DEVEL_BUILD -#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -#endif +if BUILDOPT_IS_DEVEL_BUILD +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 2da1d749..4d027555 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -337,6 +337,7 @@ ostree_repo_remote_list_collection_refs ostree_repo_remote_get_url ostree_repo_remote_get_gpg_verify ostree_repo_remote_get_gpg_verify_summary +ostree_repo_remote_get_gpg_keys ostree_repo_remote_gpg_import ostree_repo_remote_fetch_summary ostree_repo_remote_fetch_summary_with_options @@ -482,6 +483,8 @@ ostree_repo_regenerate_summary OSTREE_REPO OSTREE_IS_REPO OSTREE_TYPE_REPO +OSTREE_GPG_KEY_GVARIANT_STRING +OSTREE_GPG_KEY_GVARIANT_FORMAT ostree_repo_get_type ostree_repo_commit_modifier_get_type ostree_repo_transaction_stats_get_type diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index e3cd14a4..75bc4647 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -22,6 +22,11 @@ - uncomment the include in Makefile-libostree.am */ +LIBOSTREE_2021.4 { +global: + ostree_repo_remote_get_gpg_keys; +} LIBOSTREE_2021.3; + /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION diff --git a/src/libostree/ostree-gpg-verifier.c b/src/libostree/ostree-gpg-verifier.c index 88850d46..e9f5c5e3 100644 --- a/src/libostree/ostree-gpg-verifier.c +++ b/src/libostree/ostree-gpg-verifier.c @@ -185,6 +185,91 @@ _ostree_gpg_verifier_import_keys (OstreeGpgVerifier *self, return ret; } +gboolean +_ostree_gpg_verifier_list_keys (OstreeGpgVerifier *self, + const char * const *key_ids, + GPtrArray **out_keys, + GCancellable *cancellable, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR("GPG", error); + g_auto(gpgme_ctx_t) context = NULL; + g_autoptr(GOutputStream) pubring_stream = NULL; + g_autofree char *tmp_dir = NULL; + g_autoptr(GPtrArray) keys = NULL; + gpgme_error_t gpg_error = 0; + gboolean ret = FALSE; + + if (g_cancellable_set_error_if_cancelled (cancellable, error)) + goto out; + + context = ot_gpgme_new_ctx (NULL, error); + if (context == NULL) + goto out; + + if (!ot_gpgme_ctx_tmp_home_dir (context, &tmp_dir, &pubring_stream, + cancellable, error)) + goto out; + + if (!_ostree_gpg_verifier_import_keys (self, context, pubring_stream, + cancellable, error)) + goto out; + + keys = g_ptr_array_new_with_free_func ((GDestroyNotify) gpgme_key_unref); + if (key_ids != NULL) + { + for (guint i = 0; key_ids[i] != NULL; i++) + { + gpgme_key_t key = NULL; + + gpg_error = gpgme_get_key (context, key_ids[i], &key, 0); + if (gpg_error != GPG_ERR_NO_ERROR) + { + ot_gpgme_throw (gpg_error, error, "Unable to find key \"%s\"", + key_ids[i]); + goto out; + } + + /* Transfer ownership. */ + g_ptr_array_add (keys, key); + } + } + else + { + gpg_error = gpgme_op_keylist_start (context, NULL, 0); + while (gpg_error == GPG_ERR_NO_ERROR) + { + gpgme_key_t key = NULL; + + gpg_error = gpgme_op_keylist_next (context, &key); + if (gpg_error != GPG_ERR_NO_ERROR) + break; + + /* Transfer ownership. */ + g_ptr_array_add (keys, key); + } + + if (gpgme_err_code (gpg_error) != GPG_ERR_EOF) + { + ot_gpgme_throw (gpg_error, error, "Unable to list keys"); + goto out; + } + } + + if (out_keys != NULL) + *out_keys = g_steal_pointer (&keys); + + ret = TRUE; + + out: + if (tmp_dir != NULL) { + ot_gpgme_kill_agent (tmp_dir); + (void) glnx_shutil_rm_rf_at (AT_FDCWD, tmp_dir, NULL, NULL); + } + + return ret; +} + OstreeGpgVerifyResult * _ostree_gpg_verifier_check_signature (OstreeGpgVerifier *self, GBytes *signed_data, diff --git a/src/libostree/ostree-gpg-verifier.h b/src/libostree/ostree-gpg-verifier.h index 634d33b2..3d803c49 100644 --- a/src/libostree/ostree-gpg-verifier.h +++ b/src/libostree/ostree-gpg-verifier.h @@ -51,6 +51,12 @@ OstreeGpgVerifyResult *_ostree_gpg_verifier_check_signature (OstreeGpgVerifier * GCancellable *cancellable, GError **error); +gboolean _ostree_gpg_verifier_list_keys (OstreeGpgVerifier *self, + const char * const *key_ids, + GPtrArray **out_keys, + GCancellable *cancellable, + GError **error); + gboolean _ostree_gpg_verifier_add_keyring_dir (OstreeGpgVerifier *self, GFile *path, GCancellable *cancellable, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 254f7010..b20aa617 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2353,6 +2353,130 @@ out: #endif /* OSTREE_DISABLE_GPGME */ } +static gboolean +_ostree_repo_gpg_prepare_verifier (OstreeRepo *self, + const gchar *remote_name, + GFile *keyringdir, + GFile *extra_keyring, + gboolean add_global_keyrings, + OstreeGpgVerifier **out_verifier, + GCancellable *cancellable, + GError **error); + +/** + * ostree_repo_remote_get_gpg_keys: + * @self: an #OstreeRepo + * @name (nullable): name of the remote or %NULL + * @key_ids: (array zero-terminated=1) (element-type utf8) (nullable): + * a %NULL-terminated array of GPG key IDs to include, or %NULL + * @out_keys: (out) (optional) (element-type GVariant) (transfer container): + * return location for a #GPtrArray of the remote's trusted GPG keys, or + * %NULL + * @cancellable: (nullable): a #GCancellable, or %NULL + * @error: return location for a #GError, or %NULL + * + * Enumerate the trusted GPG keys for the remote @name. If @name is + * %NULL, the global GPG keys will be returned. The keys will be + * returned in the @out_keys #GPtrArray. Each element in the array is a + * #GVariant of format %OSTREE_GPG_KEY_GVARIANT_FORMAT. The @key_ids + * array can be used to limit which keys are included. If @key_ids is + * %NULL, then all keys are included. + * + * Returns: %TRUE if the GPG keys could be enumerated, %FALSE otherwise + * + * Since: 2021.4 + */ +gboolean +ostree_repo_remote_get_gpg_keys (OstreeRepo *self, + const char *name, + const char * const *key_ids, + GPtrArray **out_keys, + GCancellable *cancellable, + GError **error) +{ +#ifndef OSTREE_DISABLE_GPGME + g_autoptr(OstreeGpgVerifier) verifier = NULL; + gboolean global_keyrings = (name == NULL); + if (!_ostree_repo_gpg_prepare_verifier (self, name, NULL, NULL, global_keyrings, + &verifier, cancellable, error)) + return FALSE; + + g_autoptr(GPtrArray) gpg_keys = NULL; + if (!_ostree_gpg_verifier_list_keys (verifier, key_ids, &gpg_keys, + cancellable, error)) + return FALSE; + + g_autoptr(GPtrArray) keys = + g_ptr_array_new_with_free_func ((GDestroyNotify) g_variant_unref); + for (guint i = 0; i < gpg_keys->len; i++) + { + gpgme_key_t key = gpg_keys->pdata[i]; + + g_auto(GVariantBuilder) subkeys_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_builder_init (&subkeys_builder, G_VARIANT_TYPE ("a(a{sv})")); + g_auto(GVariantBuilder) uids_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_builder_init (&uids_builder, G_VARIANT_TYPE ("a(a{sv})")); + for (gpgme_subkey_t subkey = key->subkeys; subkey != NULL; + subkey = subkey->next) + { + g_auto(GVariantDict) subkey_dict = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_dict_init (&subkey_dict, NULL); + g_variant_dict_insert_value (&subkey_dict, "fingerprint", + g_variant_new_string (subkey->fpr)); + g_variant_dict_insert_value (&subkey_dict, "created", + g_variant_new_int64 (GINT64_TO_BE (subkey->timestamp))); + g_variant_dict_insert_value (&subkey_dict, "expires", + g_variant_new_int64 (GINT64_TO_BE (subkey->expires))); + g_variant_dict_insert_value (&subkey_dict, "revoked", + g_variant_new_boolean (subkey->revoked)); + g_variant_dict_insert_value (&subkey_dict, "expired", + g_variant_new_boolean (subkey->expired)); + g_variant_dict_insert_value (&subkey_dict, "invalid", + g_variant_new_boolean (subkey->invalid)); + g_variant_builder_add (&subkeys_builder, "(@a{sv})", + g_variant_dict_end (&subkey_dict)); + } + + for (gpgme_user_id_t uid = key->uids; uid != NULL; uid = uid->next) + { + g_auto(GVariantDict) uid_dict = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_dict_init (&uid_dict, NULL); + g_variant_dict_insert_value (&uid_dict, "uid", + g_variant_new_string (uid->uid)); + g_variant_dict_insert_value (&uid_dict, "name", + g_variant_new_string (uid->name)); + g_variant_dict_insert_value (&uid_dict, "comment", + g_variant_new_string (uid->comment)); + g_variant_dict_insert_value (&uid_dict, "email", + g_variant_new_string (uid->email)); + g_variant_dict_insert_value (&uid_dict, "revoked", + g_variant_new_boolean (uid->revoked)); + g_variant_dict_insert_value (&uid_dict, "invalid", + g_variant_new_boolean (uid->invalid)); + g_variant_builder_add (&uids_builder, "(@a{sv})", + g_variant_dict_end (&uid_dict)); + } + + /* Currently empty */ + g_auto(GVariantDict) metadata_dict = OT_VARIANT_BUILDER_INITIALIZER; + g_variant_dict_init (&metadata_dict, NULL); + + GVariant *key_variant = g_variant_new ("(@a(a{sv})@a(a{sv})@a{sv})", + g_variant_builder_end (&subkeys_builder), + g_variant_builder_end (&uids_builder), + g_variant_dict_end (&metadata_dict)); + g_ptr_array_add (keys, g_variant_ref_sink (key_variant)); + } + + if (out_keys) + *out_keys = g_steal_pointer (&keys); + + return TRUE; +#else /* OSTREE_DISABLE_GPGME */ + return glnx_throw (error, "GPG feature is disabled in a build time"); +#endif /* OSTREE_DISABLE_GPGME */ +} + /** * ostree_repo_remote_fetch_summary: * @self: Self diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 08d3d408..7694d40c 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1425,6 +1425,46 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, const char *name, gboolean *out_gpg_verify_summary, GError **error); + +/** + * OSTREE_GPG_KEY_GVARIANT_FORMAT: + * + * - a(a{sv}) - Array of subkeys. Each a{sv} dictionary represents a + * subkey. The primary key is the first subkey. The following keys are + * currently recognized: + * - key: `fingerprint`, value: `s`, key fingerprint hexadecimal string + * - key: `created`, value: `x`, key creation timestamp (seconds since + * the Unix epoch in UTC, big-endian) + * - key: `expires`, value: `x`, key expiration timestamp (seconds since + * the Unix epoch in UTC, big-endian). If this value is 0, the key does + * not expire. + * - key: `revoked`, value: `b`, whether key is revoked + * - key: `expired`, value: `b`, whether key is expired + * - key: `invalid`, value: `b`, whether key is invalid + * - a(a{sv}) - Array of user IDs. Each a{sv} dictionary represents a + * user ID. The following keys are currently recognized: + * - key: `uid`, value: `s`, full user ID (name, email and comment) + * - key: `name`, value: `s`, user ID name component + * - key: `comment`, value: `s`, user ID comment component + * - key: `email`, value: `s`, user ID email component + * - key: `revoked`, value: `b`, whether user ID is revoked + * - key: `invalid`, value: `b`, whether user ID is invalid + * - a{sv} - Additional metadata dictionary. There are currently no + * additional metadata keys defined. + * + * Since: 2021.4 + */ +#define OSTREE_GPG_KEY_GVARIANT_STRING "(a(a{sv})a(a{sv})a{sv})" +#define OSTREE_GPG_KEY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_GPG_KEY_GVARIANT_STRING) + +_OSTREE_PUBLIC +gboolean ostree_repo_remote_get_gpg_keys (OstreeRepo *self, + const char *name, + const char * const *key_ids, + GPtrArray **out_keys, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_remote_gpg_import (OstreeRepo *self, const char *name, From 74fb0c5f7826e90efcc905ba75cf8a176d61f0a5 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 13 Aug 2019 10:10:50 -0600 Subject: [PATCH 07/47] bin/remote: Add list-gpg-keys subcommand This provides a wrapper for the `ostree_repo_remote_get_gpg_keys` function to show the GPG keys associated with a remote. This is particularly useful for validating that GPG key updates have been applied. Tests are added, which checks the `ostree_repo_remote_get_gpg_keys` API by extension. --- Makefile-ostree.am | 1 + Makefile-tests.am | 1 + bash/ostree | 35 +++++ man/ostree-remote.xml | 9 +- src/ostree/ot-builtin-remote.c | 3 + src/ostree/ot-dump.c | 113 ++++++++++++++- src/ostree/ot-dump.h | 3 + src/ostree/ot-remote-builtin-list-gpg-keys.c | 66 +++++++++ src/ostree/ot-remote-builtins.h | 1 + tests/test-remote-list-gpg-keys.sh | 144 +++++++++++++++++++ 10 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 src/ostree/ot-remote-builtin-list-gpg-keys.c create mode 100755 tests/test-remote-list-gpg-keys.sh diff --git a/Makefile-ostree.am b/Makefile-ostree.am index fd5ec9de..a5509f7c 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -105,6 +105,7 @@ ostree_SOURCES += \ if USE_GPGME ostree_SOURCES += \ src/ostree/ot-remote-builtin-gpg-import.c \ + src/ostree/ot-remote-builtin-list-gpg-keys.c \ $(NULL) endif diff --git a/Makefile-tests.am b/Makefile-tests.am index 295c734e..1997bfd8 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -152,6 +152,7 @@ _installed_or_uninstalled_test_scripts = \ if USE_GPGME _installed_or_uninstalled_test_scripts += \ tests/test-remote-gpg-import.sh \ + tests/test-remote-list-gpg-keys.sh \ tests/test-gpg-signed-commit.sh \ tests/test-admin-gpg.sh \ $(NULL) diff --git a/bash/ostree b/bash/ostree index d1de8530..32d5e317 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1235,6 +1235,40 @@ _ostree_remote_list_cookies() { return 0 } +_ostree_remote_list_gpg_keys() { + local boolean_options=" + $main_boolean_options + " + + local options_with_args=" + --repo + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --repo) + __ostree_compreply_dirs_only + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + *) + local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) + + if [ $cword -eq $argpos ]; then + __ostree_compreply_remotes + fi + esac + + return 0 +} + _ostree_remote_refs() { local boolean_options=" $main_boolean_options @@ -1349,6 +1383,7 @@ _ostree_remote() { gpg-import list list-cookies + list-gpg-keys refs show-url summary diff --git a/man/ostree-remote.xml b/man/ostree-remote.xml index 407f7e3d..928bf9b5 100644 --- a/man/ostree-remote.xml +++ b/man/ostree-remote.xml @@ -65,6 +65,9 @@ Boston, MA 02111-1307, USA. ostree remote gpg-import OPTIONS NAME KEY-ID + + ostree remote list-gpg-keys NAME + ostree remote refs NAME @@ -106,7 +109,11 @@ Boston, MA 02111-1307, USA. for more information. - The gpg-import subcommand can associate GPG keys to a specific remote repository for use when pulling signed commits from that repository (if GPG verification is enabled). + The gpg-import subcommand can associate GPG + keys to a specific remote repository for use when pulling signed + commits from that repository (if GPG verification is enabled). The + list-gpg-keys subcommand can be used to see the + GPG keys currently associated with a remote repository. The GPG keys to import may be in binary OpenPGP format or ASCII armored. The optional KEY-ID list can restrict which keys are imported from a keyring file or input stream. All keys are imported if this list is omitted. If neither nor options are given, then keys are imported from the user's personal GPG keyring. diff --git a/src/ostree/ot-builtin-remote.c b/src/ostree/ot-builtin-remote.c index 6b3f6a26..7028eacc 100644 --- a/src/ostree/ot-builtin-remote.c +++ b/src/ostree/ot-builtin-remote.c @@ -44,6 +44,9 @@ static OstreeCommand remote_subcommands[] = { { "gpg-import", OSTREE_BUILTIN_FLAG_NONE, ot_remote_builtin_gpg_import, "Import GPG keys" }, + { "list-gpg-keys", OSTREE_BUILTIN_FLAG_NONE, + ot_remote_builtin_list_gpg_keys, + "Show remote GPG keys" }, #endif /* OSTREE_DISABLE_GPGME */ #ifdef HAVE_LIBCURL_OR_LIBSOUP { "add-cookie", OSTREE_BUILTIN_FLAG_NONE, diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index a8ed54a2..1c0f04a9 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -53,6 +53,7 @@ ot_dump_variant (GVariant *variant) static gchar * format_timestamp (guint64 timestamp, + gboolean local_tz, GError **error) { GDateTime *dt; @@ -66,7 +67,19 @@ format_timestamp (guint64 timestamp, return NULL; } - str = g_date_time_format (dt, "%Y-%m-%d %H:%M:%S +0000"); + if (local_tz) + { + /* Convert to local time and display in the locale's preferred + * representation. + */ + g_autoptr(GDateTime) dt_local = g_date_time_to_local (dt); + str = g_date_time_format (dt_local, "%c"); + } + else + { + str = g_date_time_format (dt, "%Y-%m-%d %H:%M:%S +0000"); + } + g_date_time_unref (dt); return str; @@ -124,7 +137,7 @@ dump_commit (GVariant *variant, &subject, &body, ×tamp, NULL, NULL); timestamp = GUINT64_FROM_BE (timestamp); - str = format_timestamp (timestamp, &local_error); + str = format_timestamp (timestamp, FALSE, &local_error); if (!str) { g_assert (local_error); /* Pacify static analysis */ @@ -390,3 +403,99 @@ ot_dump_summary_bytes (GBytes *summary_bytes, g_print ("%s: %s\n", key, value_str); } } + +static gboolean +dump_gpg_subkey (GVariant *subkey, + gboolean primary, + GError **error) +{ + const gchar *fingerprint = NULL; + gint64 created = 0; + gint64 expires = 0; + gboolean revoked = FALSE; + gboolean expired = FALSE; + gboolean invalid = FALSE; + (void) g_variant_lookup (subkey, "fingerprint", "&s", &fingerprint); + (void) g_variant_lookup (subkey, "created", "x", &created); + (void) g_variant_lookup (subkey, "expires", "x", &expires); + (void) g_variant_lookup (subkey, "revoked", "b", &revoked); + (void) g_variant_lookup (subkey, "expired", "b", &expired); + (void) g_variant_lookup (subkey, "invalid", "b", &invalid); + + /* Convert timestamps from big endian if needed */ + created = GINT64_FROM_BE (created); + expires = GINT64_FROM_BE (expires); + + g_print ("%s: %s%s%s\n", + primary ? "Key" : " Subkey", + fingerprint, + revoked ? " (revoked)" : "", + invalid ? " (invalid)" : ""); + + g_autofree gchar *created_str = format_timestamp (created, TRUE, + error); + if (created_str == NULL) + return FALSE; + g_print ("%sCreated: %s\n", + primary ? " " : " ", + created_str); + + if (expires > 0) + { + g_autofree gchar *expires_str = format_timestamp (expires, TRUE, + error); + if (expires_str == NULL) + return FALSE; + g_print ("%s%s: %s\n", + primary ? " " : " ", + expired ? "Expired" : "Expires", + expires_str); + } + + return TRUE; +} + +gboolean +ot_dump_gpg_key (GVariant *key, + GError **error) +{ + if (!g_variant_is_of_type (key, OSTREE_GPG_KEY_GVARIANT_FORMAT)) + return glnx_throw (error, "GPG key variant type doesn't match '%s'", + OSTREE_GPG_KEY_GVARIANT_STRING); + + g_autoptr(GVariant) subkeys_v = g_variant_get_child_value (key, 0); + GVariantIter subkeys_iter; + g_variant_iter_init (&subkeys_iter, subkeys_v); + + g_autoptr(GVariant) primary_key = NULL; + g_variant_iter_next (&subkeys_iter, "(@a{sv})", &primary_key); + if (!dump_gpg_subkey (primary_key, TRUE, error)) + return FALSE; + + g_autoptr(GVariant) uids_v = g_variant_get_child_value (key, 1); + GVariantIter uids_iter; + g_variant_iter_init (&uids_iter, uids_v); + GVariant *uid_v = NULL; + while (g_variant_iter_loop (&uids_iter, "(@a{sv})", &uid_v)) + { + const gchar *uid = NULL; + gboolean revoked = FALSE; + gboolean invalid = FALSE; + (void) g_variant_lookup (uid_v, "uid", "&s", &uid); + (void) g_variant_lookup (uid_v, "revoked", "b", &revoked); + (void) g_variant_lookup (uid_v, "invalid", "b", &invalid); + g_print (" UID: %s%s%s\n", + uid, + revoked ? " (revoked)" : "", + invalid ? " (invalid)" : ""); + } + + GVariant *subkey = NULL; + while (g_variant_iter_loop (&subkeys_iter, "(@a{sv})", &subkey)) + { + if (!dump_gpg_subkey (subkey, FALSE, error)) + return FALSE; + } + + return TRUE; +} diff --git a/src/ostree/ot-dump.h b/src/ostree/ot-dump.h index 0e1952af..02e2f1a6 100644 --- a/src/ostree/ot-dump.h +++ b/src/ostree/ot-dump.h @@ -42,3 +42,6 @@ void ot_dump_object (OstreeObjectType objtype, void ot_dump_summary_bytes (GBytes *summary_bytes, OstreeDumpFlags flags); + +gboolean ot_dump_gpg_key (GVariant *key, + GError **error); diff --git a/src/ostree/ot-remote-builtin-list-gpg-keys.c b/src/ostree/ot-remote-builtin-list-gpg-keys.c new file mode 100644 index 00000000..84d0f1a3 --- /dev/null +++ b/src/ostree/ot-remote-builtin-list-gpg-keys.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2015 Red Hat, Inc. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "otutil.h" + +#include "ot-main.h" +#include "ot-dump.h" +#include "ot-remote-builtins.h" + +/* ATTENTION: + * Please remember to update the bash-completion script (bash/ostree) and + * man page (man/ostree-remote.xml) when changing the option list. + */ + +static GOptionEntry option_entries[] = { + { NULL } +}; + +gboolean +ot_remote_builtin_list_gpg_keys (int argc, + char **argv, + OstreeCommandInvocation *invocation, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GOptionContext) context = g_option_context_new ("NAME"); + g_autoptr(OstreeRepo) repo = NULL; + if (!ostree_option_context_parse (context, option_entries, &argc, &argv, + invocation, &repo, cancellable, error)) + return FALSE; + + const char *remote_name = (argc > 1) ? argv[1] : NULL; + + g_autoptr(GPtrArray) keys = NULL; + if (!ostree_repo_remote_get_gpg_keys (repo, remote_name, NULL, &keys, + cancellable, error)) + return FALSE; + + for (guint i = 0; i < keys->len; i++) + { + if (!ot_dump_gpg_key (keys->pdata[i], error)) + return FALSE; + } + + return TRUE; +} diff --git a/src/ostree/ot-remote-builtins.h b/src/ostree/ot-remote-builtins.h index 71b2365a..4b46af19 100644 --- a/src/ostree/ot-remote-builtins.h +++ b/src/ostree/ot-remote-builtins.h @@ -32,6 +32,7 @@ G_BEGIN_DECLS BUILTINPROTO(add); BUILTINPROTO(delete); BUILTINPROTO(gpg_import); +BUILTINPROTO(list_gpg_keys); BUILTINPROTO(list); #ifdef HAVE_LIBCURL_OR_LIBSOUP BUILTINPROTO(add_cookie); diff --git a/tests/test-remote-list-gpg-keys.sh b/tests/test-remote-list-gpg-keys.sh new file mode 100755 index 00000000..5ad6c9f2 --- /dev/null +++ b/tests/test-remote-list-gpg-keys.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# Copyright © 2021 Endless OS Foundation LLC +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +# We don't want OSTREE_GPG_HOME used for most of these tests. +emptydir=${test_tmpdir}/empty +trusteddir=${OSTREE_GPG_HOME} +mkdir ${emptydir} +OSTREE_GPG_HOME=${emptydir} + +# Key listings show dates using the local timezone, so specify UTC for +# consistency. +export TZ=UTC + +# Some tests require an appropriate gpg +num_non_gpg_tests=5 +num_gpg_tests=2 +num_tests=$((num_non_gpg_tests + num_gpg_tests)) + +echo "1..${num_tests}" + +setup_test_repository "archive" + +cd ${test_tmpdir} +${OSTREE} remote add R1 http://example.com/repo + +# No remote keyring should list no keys. +${OSTREE} remote list-gpg-keys R1 > result +assert_file_empty result + +echo "ok remote no keyring" + +# Make the global keyring available and make sure there are still no +# keys found for a specified remote. +OSTREE_GPG_HOME=${trusteddir} +${OSTREE} remote list-gpg-keys R1 > result +OSTREE_GPG_HOME=${emptydir} +assert_file_empty result + +echo "ok remote with global keyring" + +# Import a key and check that it's listed +${OSTREE} remote gpg-import --keyring ${TEST_GPG_KEYHOME}/key1.asc R1 +${OSTREE} remote list-gpg-keys R1 > result +cat > expected <<"EOF" +Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA + Created: Tue Sep 10 02:29:42 2013 + UID: Ostree Tester + Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 + Created: Tue Sep 10 02:29:42 2013 +EOF +assert_files_equal result expected + +echo "ok remote with keyring" + +# Check the global keys with no keyring +OSTREE_GPG_HOME=${emptydir} +${OSTREE} remote list-gpg-keys > result +assert_file_empty result + +echo "ok global no keyring" + +# Now check the global keys with a keyring +OSTREE_GPG_HOME=${trusteddir} +${OSTREE} remote list-gpg-keys > result +OSTREE_GPG_HOME=${emptydir} +cat > expected <<"EOF" +Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA + Created: Tue Sep 10 02:29:42 2013 + UID: Ostree Tester + Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 + Created: Tue Sep 10 02:29:42 2013 +Key: 7B3B1020D74479687FDB2273D8228CFECA950D41 + Created: Tue Mar 17 14:00:32 2015 + UID: Ostree Tester II + Subkey: 1EFA95C06EB1EB91754575E004B69C2560D53993 + Created: Tue Mar 17 14:00:32 2015 +Key: 7D29CF060B8269CDF63BFBDD0D15FAE7DF444D67 + Created: Tue Mar 17 14:01:05 2015 + UID: Ostree Tester III + Subkey: 0E45E48CBF7B360C0E04443E0C601A7402416340 + Created: Tue Mar 17 14:01:05 2015 +EOF +assert_files_equal result expected + +echo "ok global with keyring" + +# Tests checking for expiration and revocation listings require gpg. +GPG=$(which_gpg) +if [ -z "${GPG}" ]; then + # Print a skip message per skipped test + for (( i = 0; i < num_gpg_tests; i++ )); do + echo "ok # SKIP this test requires gpg" + done +else + # The GPG private keyring in gpghome is in the older secring.gpg + # format, but we're likely using a newer gpg. Normally it's + # implicitly migrated to the newer format, but this test hasn't + # signed anything, so the private keys haven't been loaded. Force + # the migration by listing the private keys. + ${GPG} --homedir=${test_tmpdir}/gpghome -K >/dev/null + + # Expire key1, wait for it to be expired and re-import it. + ${GPG} --homedir=${test_tmpdir}/gpghome --quick-set-expire ${TEST_GPG_KEYFPR_1} seconds=1 + sleep 2 + ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1expired.asc + ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1expired.asc R1 + ${OSTREE} remote list-gpg-keys R1 > result + assert_file_has_content result "^ Expired:" + + echo "ok remote expired key" + + # Revoke key1 and re-import it. + ${GPG} --homedir=${TEST_GPG_KEYHOME} --import ${TEST_GPG_KEYHOME}/revocations/key1.rev + ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1revoked.asc + ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1revoked.asc R1 + ${OSTREE} remote list-gpg-keys R1 > result + assert_file_has_content result "^Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA (revoked)" + assert_file_has_content result "^ UID: Ostree Tester (revoked)" + assert_file_has_content result "^ Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 (revoked)" + + echo "ok remote revoked key" +fi From fbff05e28d0e0d3c8424a7d4a5652a253a553b04 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 26 Aug 2019 10:09:57 -0600 Subject: [PATCH 08/47] libotutil: Import implementation of zbase32 encoding This will be used to implement the PGP Web Key Directory (WKD) URL generation. This is a slightly cleaned up implementation[1] taken from the zbase32 author's original implementation[2]. It provides a single zbase32_encode API to convert a set of bytes to the zbase32 encoding. I believe this should be acceptable for inclusion in ostree. The license in the source files is BSD style while the original repo LICENSE file claims the Creative Commons CC0 1.0 Universal license, which is public domain. 1. https://github.com/dbnicholson/libbase32/tree/for-ostree 2. https://github.com/zooko/libbase32 --- Makefile-otutil.am | 2 + src/libotutil/zbase32.c | 141 ++++++++++++++++++++++++++++++++++++++++ src/libotutil/zbase32.h | 49 ++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 src/libotutil/zbase32.c create mode 100644 src/libotutil/zbase32.h diff --git a/Makefile-otutil.am b/Makefile-otutil.am index e8901b57..7bc87b6a 100644 --- a/Makefile-otutil.am +++ b/Makefile-otutil.am @@ -49,6 +49,8 @@ if USE_GPGME libotutil_la_SOURCES += \ src/libotutil/ot-gpg-utils.c \ src/libotutil/ot-gpg-utils.h \ + src/libotutil/zbase32.c \ + src/libotutil/zbase32.h \ $(NULL) endif diff --git a/src/libotutil/zbase32.c b/src/libotutil/zbase32.c new file mode 100644 index 00000000..39fa97a4 --- /dev/null +++ b/src/libotutil/zbase32.c @@ -0,0 +1,141 @@ +/** + * copyright 2002, 2003 Bryce "Zooko" Wilcox-O'Hearn + * mailto:zooko@zooko.com + * + * See the end of this file for the free software, open source license (BSD-style). + */ +#include "zbase32.h" + +#include +#include +#include +#include /* XXX only for debug printfs */ + +static const char*const chars="ybndrfg8ejkmcpqxot1uwisza345h769"; + +/* Types from zstr */ +/** + * A zstr is simply an unsigned int length and a pointer to a buffer of + * unsigned chars. + */ +typedef struct { + size_t len; /* the length of the string (not counting the null-terminating character) */ + unsigned char* buf; /* pointer to the first byte */ +} zstr; + +/** + * A zstr is simply an unsigned int length and a pointer to a buffer of + * const unsigned chars. + */ +typedef struct { + size_t len; /* the length of the string (not counting the null-terminating character) */ + const unsigned char* buf; /* pointer to the first byte */ +} czstr; + +/* Functions from zstr */ +static zstr +new_z(const size_t len) +{ + zstr result; + result.buf = (unsigned char *)malloc(len+1); + if (result.buf == NULL) { + result.len = 0; + return result; + } + result.len = len; + result.buf[len] = '\0'; + return result; +} + +/* Functions from zutil */ +static size_t +divceil(size_t n, size_t d) +{ + return n/d+((n%d)!=0); +} + +static zstr b2a_l_extra_Duffy(const czstr os, const size_t lengthinbits) +{ + zstr result = new_z(divceil(os.len*8, 5)); /* if lengthinbits is not a multiple of 8 then this is allocating space for 0, 1, or 2 extra quintets that will be truncated at the end of this function if they are not needed */ + if (result.buf == NULL) + return result; + + unsigned char* resp = result.buf + result.len; /* pointer into the result buffer, initially pointing to the "one-past-the-end" quintet */ + const unsigned char* osp = os.buf + os.len; /* pointer into the os buffer, initially pointing to the "one-past-the-end" octet */ + + /* Now this is a real live Duff's device. You gotta love it. */ + unsigned long x=0; /* to hold up to 32 bits worth of the input */ + switch ((osp - os.buf) % 5) { + case 0: + do { + x = *--osp; + *--resp = chars[x % 32]; /* The least sig 5 bits go into the final quintet. */ + x /= 32; /* ... now we have 3 bits worth in x... */ + case 4: + x |= ((unsigned long)(*--osp)) << 3; /* ... now we have 11 bits worth in x... */ + *--resp = chars[x % 32]; + x /= 32; /* ... now we have 6 bits worth in x... */ + *--resp = chars[x % 32]; + x /= 32; /* ... now we have 1 bits worth in x... */ + case 3: + x |= ((unsigned long)(*--osp)) << 1; /* The 8 bits from the 2-indexed octet. So now we have 9 bits worth in x... */ + *--resp = chars[x % 32]; + x /= 32; /* ... now we have 4 bits worth in x... */ + case 2: + x |= ((unsigned long)(*--osp)) << 4; /* The 8 bits from the 1-indexed octet. So now we have 12 bits worth in x... */ + *--resp = chars[x%32]; + x /= 32; /* ... now we have 7 bits worth in x... */ + *--resp = chars[x%32]; + x /= 32; /* ... now we have 2 bits worth in x... */ + case 1: + x |= ((unsigned long)(*--osp)) << 2; /* The 8 bits from the 0-indexed octet. So now we have 10 bits worth in x... */ + *--resp = chars[x%32]; + x /= 32; /* ... now we have 5 bits worth in x... */ + *--resp = chars[x]; + } while (osp > os.buf); + } /* switch ((osp - os.buf) % 5) */ + + /* truncate any unused trailing zero quintets */ + result.len = divceil(lengthinbits, 5); + result.buf[result.len] = '\0'; + return result; +} + +static zstr b2a_l(const czstr os, const size_t lengthinbits) +{ + return b2a_l_extra_Duffy(os, lengthinbits); +} + +static zstr b2a(const czstr os) +{ + return b2a_l(os, os.len*8); +} + +char * +zbase32_encode(const unsigned char *data, size_t length) +{ + czstr input = { length, data }; + zstr output = b2a(input); + return (char *)output.buf; +} + +/** + * Copyright (c) 2002 Bryce "Zooko" Wilcox-O'Hearn + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software to deal in this software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this software, and to permit + * persons to whom this software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of this software. + * + * THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THIS SOFTWARE. + */ diff --git a/src/libotutil/zbase32.h b/src/libotutil/zbase32.h new file mode 100644 index 00000000..bf9cf683 --- /dev/null +++ b/src/libotutil/zbase32.h @@ -0,0 +1,49 @@ +/** + * copyright 2002, 2003 Bryce "Zooko" Wilcox-O'Hearn + * mailto:zooko@zooko.com + * + * See the end of this file for the free software, open source license (BSD-style). + */ +#ifndef __INCL_base32_h +#define __INCL_base32_h + +static char const* const base32_h_cvsid = "$Id: base32.h,v 1.11 2003/12/15 01:16:19 zooko Exp $"; + +static int const base32_vermaj = 0; +static int const base32_vermin = 9; +static int const base32_vermicro = 12; +static char const* const base32_vernum = "0.9.12"; + +#include +#include + +/** + * @param data to be zbase-32 encoded + * @param length size of the data buffer + * + * @return an allocated string containing the zbase-32 encoded representation + */ +char *zbase32_encode(const unsigned char *data, size_t length); + +#endif /* #ifndef __INCL_base32_h */ + +/** + * Copyright (c) 2002 Bryce "Zooko" Wilcox-O'Hearn + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software to deal in this software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of this software, and to permit + * persons to whom this software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of this software. + * + * THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THIS SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THIS SOFTWARE. + */ From 4fa403aee500df058a7d3ee2b6b1db4300ad7a92 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Tue, 27 Aug 2019 10:28:10 -0600 Subject: [PATCH 09/47] libotutil: Add helper for GPG WKD update URLs Calculate the advanced and direct update URLs for the key discovery portion[1] of the OpenPGP Web Key Directory specification, and include the URLs in the key listing in ostree_repo_remote_get_gpg_keys(). These URLs can be used to locate updated GPG keys for the remote. 1. https://datatracker.ietf.org/doc/html/draft-koch-openpgp-webkey-service#section-3.1 --- src/libotutil/ot-gpg-utils.c | 75 ++++++++++++++++++++++++++++++++++++ src/libotutil/ot-gpg-utils.h | 5 +++ 2 files changed, 80 insertions(+) diff --git a/src/libotutil/ot-gpg-utils.c b/src/libotutil/ot-gpg-utils.c index 743d941e..4dbefdbd 100644 --- a/src/libotutil/ot-gpg-utils.c +++ b/src/libotutil/ot-gpg-utils.c @@ -27,6 +27,7 @@ #include #include "libglnx.h" +#include "zbase32.h" /* Like glnx_throw_errno_prefix, but takes @gpg_error */ gboolean @@ -538,3 +539,77 @@ ot_gpgme_kill_agent (const char *homedir) return; } } + +/* Takes the SHA1 checksum of the local component of an email address and + * returns the zbase32 encoding. + */ +static char * +encode_wkd_local (const char *local) +{ + g_return_val_if_fail (local != NULL, NULL); + + guint8 digest[20] = { 0 }; + gsize len = sizeof (digest); + g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA1); + g_checksum_update (checksum, (const guchar *)local, -1); + g_checksum_get_digest (checksum, digest, &len); + + char *encoded = zbase32_encode (digest, len); + + /* If the returned string is NULL, then there must have been a memory + * allocation problem. Just exit immediately like g_malloc. + */ + if (encoded == NULL) + g_error ("%s: %s", G_STRLOC, g_strerror (errno)); + + return encoded; +} + +/* Implementation of OpenPGP Web Key Directory URLs as defined in + * https://datatracker.ietf.org/doc/html/draft-koch-openpgp-webkey-service + */ +gboolean +ot_gpg_wkd_urls (const char *email, + char **out_advanced_url, + char **out_direct_url, + GError **error) +{ + g_return_val_if_fail (email != NULL, FALSE); + + g_auto(GStrv) email_parts = g_strsplit (email, "@", -1); + if (g_strv_length (email_parts) != 2) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT, + "Invalid email address \"%s\"", email); + return FALSE; + } + + g_autofree char *local_lowered = g_ascii_strdown (email_parts[0], -1); + g_autofree char *domain_lowered = g_ascii_strdown (email_parts[1], -1); + g_autofree char *local_encoded = encode_wkd_local (local_lowered); + g_autofree char *local_escaped = g_uri_escape_string (email_parts[0], NULL, FALSE); + + g_autofree char *advanced_url = g_strdup_printf ("https://openpgpkey.%s" + "/.well-known/openpgpkey" + "/%s/hu/%s?l=%s", + email_parts[1], + domain_lowered, + local_encoded, + local_escaped); + g_debug ("GPG UID \"%s\" advanced WKD URL: %s", email, advanced_url); + + g_autofree char *direct_url = g_strdup_printf ("https://%s" + "/.well-known/openpgpkey" + "/hu/%s?l=%s", + email_parts[1], + local_encoded, + local_escaped); + g_debug ("GPG UID \"%s\" direct WKD URL: %s", email, direct_url); + + if (out_advanced_url != NULL) + *out_advanced_url = g_steal_pointer (&advanced_url); + if (out_direct_url != NULL) + *out_direct_url = g_steal_pointer (&direct_url); + + return TRUE; +} diff --git a/src/libotutil/ot-gpg-utils.h b/src/libotutil/ot-gpg-utils.h index e8a240b5..b559b695 100644 --- a/src/libotutil/ot-gpg-utils.h +++ b/src/libotutil/ot-gpg-utils.h @@ -48,4 +48,9 @@ gpgme_ctx_t ot_gpgme_new_ctx (const char *homedir, void ot_gpgme_kill_agent (const char *homedir); +gboolean ot_gpg_wkd_urls (const char *email, + char **out_advanced_url, + char **out_direct_url, + GError **error); + G_END_DECLS From 27dc5d7d389956befd9f2de7a956073899c264de Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 26 Aug 2019 11:15:25 -0600 Subject: [PATCH 10/47] lib/repo: Include WKD update URLs in GPG key listing If the key UID contains a valid email address, include the GPG WKD update URLs in GVariant returned by ostree_repo_remote_get_gpg_keys(). --- src/libostree/ostree-repo.c | 14 ++++++++++++++ src/libostree/ostree-repo.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b20aa617..5bd76e91 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2439,6 +2439,16 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, for (gpgme_user_id_t uid = key->uids; uid != NULL; uid = uid->next) { + /* Get WKD update URLs if address set */ + g_autofree char *advanced_url = NULL; + g_autofree char *direct_url = NULL; + if (uid->address != NULL) + { + if (!ot_gpg_wkd_urls (uid->address, &advanced_url, &direct_url, + error)) + return FALSE; + } + g_auto(GVariantDict) uid_dict = OT_VARIANT_BUILDER_INITIALIZER; g_variant_dict_init (&uid_dict, NULL); g_variant_dict_insert_value (&uid_dict, "uid", @@ -2453,6 +2463,10 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, g_variant_new_boolean (uid->revoked)); g_variant_dict_insert_value (&uid_dict, "invalid", g_variant_new_boolean (uid->invalid)); + g_variant_dict_insert_value (&uid_dict, "advanced_url", + g_variant_new ("ms", advanced_url)); + g_variant_dict_insert_value (&uid_dict, "direct_url", + g_variant_new ("ms", direct_url)); g_variant_builder_add (&uids_builder, "(@a{sv})", g_variant_dict_end (&uid_dict)); } diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 7694d40c..ecf92772 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1449,6 +1449,8 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, * - key: `email`, value: `s`, user ID email component * - key: `revoked`, value: `b`, whether user ID is revoked * - key: `invalid`, value: `b`, whether user ID is invalid + * - key: `advanced_url`, value: `ms`, advanced WKD update URL + * - key: `direct_url`, value: `ms`, direct WKD update URL * - a{sv} - Additional metadata dictionary. There are currently no * additional metadata keys defined. * From 90a3bda1f8298e149cb4940369cec4bdc4e53972 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 26 Aug 2019 11:27:54 -0600 Subject: [PATCH 11/47] bin/remote: Include update URLs in list-gpg-keys --- src/ostree/ot-dump.c | 7 +++++++ tests/test-remote-list-gpg-keys.sh | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 1c0f04a9..6db6dc8f 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -488,6 +488,13 @@ ot_dump_gpg_key (GVariant *key, uid, revoked ? " (revoked)" : "", invalid ? " (invalid)" : ""); + + const char *advanced_url = NULL; + const char *direct_url = NULL; + (void) g_variant_lookup (uid_v, "advanced_url", "m&s", &advanced_url); + (void) g_variant_lookup (uid_v, "direct_url", "m&s", &direct_url); + g_print (" Advanced update URL: %s\n", advanced_url ?: ""); + g_print (" Direct update URL: %s\n", direct_url ?: ""); } GVariant *subkey = NULL; diff --git a/tests/test-remote-list-gpg-keys.sh b/tests/test-remote-list-gpg-keys.sh index 5ad6c9f2..81699f14 100755 --- a/tests/test-remote-list-gpg-keys.sh +++ b/tests/test-remote-list-gpg-keys.sh @@ -67,6 +67,8 @@ cat > expected <<"EOF" Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA Created: Tue Sep 10 02:29:42 2013 UID: Ostree Tester + Advanced update URL: https://openpgpkey.test.com/.well-known/openpgpkey/test.com/hu/iffe93qcsgp4c8ncbb378rxjo6cn9q6u?l=test + Direct update URL: https://test.com/.well-known/openpgpkey/hu/iffe93qcsgp4c8ncbb378rxjo6cn9q6u?l=test Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 Created: Tue Sep 10 02:29:42 2013 EOF @@ -89,16 +91,22 @@ cat > expected <<"EOF" Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA Created: Tue Sep 10 02:29:42 2013 UID: Ostree Tester + Advanced update URL: https://openpgpkey.test.com/.well-known/openpgpkey/test.com/hu/iffe93qcsgp4c8ncbb378rxjo6cn9q6u?l=test + Direct update URL: https://test.com/.well-known/openpgpkey/hu/iffe93qcsgp4c8ncbb378rxjo6cn9q6u?l=test Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 Created: Tue Sep 10 02:29:42 2013 Key: 7B3B1020D74479687FDB2273D8228CFECA950D41 Created: Tue Mar 17 14:00:32 2015 UID: Ostree Tester II + Advanced update URL: https://openpgpkey.test.com/.well-known/openpgpkey/test.com/hu/nnxwsxno46ap6hw7fgphp68j76egpfa9?l=test2 + Direct update URL: https://test.com/.well-known/openpgpkey/hu/nnxwsxno46ap6hw7fgphp68j76egpfa9?l=test2 Subkey: 1EFA95C06EB1EB91754575E004B69C2560D53993 Created: Tue Mar 17 14:00:32 2015 Key: 7D29CF060B8269CDF63BFBDD0D15FAE7DF444D67 Created: Tue Mar 17 14:01:05 2015 UID: Ostree Tester III + Advanced update URL: https://openpgpkey.test.com/.well-known/openpgpkey/test.com/hu/8494gyqhmrcs6gn38tn6kgjexet117cj?l=test3 + Direct update URL: https://test.com/.well-known/openpgpkey/hu/8494gyqhmrcs6gn38tn6kgjexet117cj?l=test3 Subkey: 0E45E48CBF7B360C0E04443E0C601A7402416340 Created: Tue Mar 17 14:01:05 2015 EOF From 30c054b521920a1c30f2530eabc2811230d2f97d Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 15 Jul 2021 16:24:36 -0600 Subject: [PATCH 12/47] fixup! lib/repo: Add ostree_repo_remote_get_gpg_keys() --- src/libostree/ostree-repo.c | 10 +++++----- src/libostree/ostree-repo.h | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5bd76e91..d4c50440 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2413,9 +2413,9 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, gpgme_key_t key = gpg_keys->pdata[i]; g_auto(GVariantBuilder) subkeys_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_variant_builder_init (&subkeys_builder, G_VARIANT_TYPE ("a(a{sv})")); + g_variant_builder_init (&subkeys_builder, G_VARIANT_TYPE ("aa{sv}")); g_auto(GVariantBuilder) uids_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_variant_builder_init (&uids_builder, G_VARIANT_TYPE ("a(a{sv})")); + g_variant_builder_init (&uids_builder, G_VARIANT_TYPE ("aa{sv}")); for (gpgme_subkey_t subkey = key->subkeys; subkey != NULL; subkey = subkey->next) { @@ -2433,7 +2433,7 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, g_variant_new_boolean (subkey->expired)); g_variant_dict_insert_value (&subkey_dict, "invalid", g_variant_new_boolean (subkey->invalid)); - g_variant_builder_add (&subkeys_builder, "(@a{sv})", + g_variant_builder_add (&subkeys_builder, "@a{sv}", g_variant_dict_end (&subkey_dict)); } @@ -2467,7 +2467,7 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, g_variant_new ("ms", advanced_url)); g_variant_dict_insert_value (&uid_dict, "direct_url", g_variant_new ("ms", direct_url)); - g_variant_builder_add (&uids_builder, "(@a{sv})", + g_variant_builder_add (&uids_builder, "@a{sv}", g_variant_dict_end (&uid_dict)); } @@ -2475,7 +2475,7 @@ ostree_repo_remote_get_gpg_keys (OstreeRepo *self, g_auto(GVariantDict) metadata_dict = OT_VARIANT_BUILDER_INITIALIZER; g_variant_dict_init (&metadata_dict, NULL); - GVariant *key_variant = g_variant_new ("(@a(a{sv})@a(a{sv})@a{sv})", + GVariant *key_variant = g_variant_new ("(@aa{sv}@aa{sv}@a{sv})", g_variant_builder_end (&subkeys_builder), g_variant_builder_end (&uids_builder), g_variant_dict_end (&metadata_dict)); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index ecf92772..5f3093df 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1429,7 +1429,7 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, /** * OSTREE_GPG_KEY_GVARIANT_FORMAT: * - * - a(a{sv}) - Array of subkeys. Each a{sv} dictionary represents a + * - aa{sv} - Array of subkeys. Each a{sv} dictionary represents a * subkey. The primary key is the first subkey. The following keys are * currently recognized: * - key: `fingerprint`, value: `s`, key fingerprint hexadecimal string @@ -1441,7 +1441,7 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, * - key: `revoked`, value: `b`, whether key is revoked * - key: `expired`, value: `b`, whether key is expired * - key: `invalid`, value: `b`, whether key is invalid - * - a(a{sv}) - Array of user IDs. Each a{sv} dictionary represents a + * - aa{sv} - Array of user IDs. Each a{sv} dictionary represents a * user ID. The following keys are currently recognized: * - key: `uid`, value: `s`, full user ID (name, email and comment) * - key: `name`, value: `s`, user ID name component @@ -1456,7 +1456,7 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, * * Since: 2021.4 */ -#define OSTREE_GPG_KEY_GVARIANT_STRING "(a(a{sv})a(a{sv})a{sv})" +#define OSTREE_GPG_KEY_GVARIANT_STRING "(aa{sv}aa{sv}a{sv})" #define OSTREE_GPG_KEY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_GPG_KEY_GVARIANT_STRING) _OSTREE_PUBLIC From 814e481fffa58a5b4417c8162fa4dd60db5fa3d4 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 15 Jul 2021 16:25:13 -0600 Subject: [PATCH 13/47] fixup! bin/remote: Add list-gpg-keys subcommand --- src/ostree/ot-dump.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 6db6dc8f..05c09780 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -468,7 +468,7 @@ ot_dump_gpg_key (GVariant *key, g_variant_iter_init (&subkeys_iter, subkeys_v); g_autoptr(GVariant) primary_key = NULL; - g_variant_iter_next (&subkeys_iter, "(@a{sv})", &primary_key); + g_variant_iter_next (&subkeys_iter, "@a{sv}", &primary_key); if (!dump_gpg_subkey (primary_key, TRUE, error)) return FALSE; @@ -476,7 +476,7 @@ ot_dump_gpg_key (GVariant *key, GVariantIter uids_iter; g_variant_iter_init (&uids_iter, uids_v); GVariant *uid_v = NULL; - while (g_variant_iter_loop (&uids_iter, "(@a{sv})", &uid_v)) + while (g_variant_iter_loop (&uids_iter, "@a{sv}", &uid_v)) { const gchar *uid = NULL; gboolean revoked = FALSE; @@ -498,7 +498,7 @@ ot_dump_gpg_key (GVariant *key, } GVariant *subkey = NULL; - while (g_variant_iter_loop (&subkeys_iter, "(@a{sv})", &subkey)) + while (g_variant_iter_loop (&subkeys_iter, "@a{sv}", &subkey)) { if (!dump_gpg_subkey (subkey, FALSE, error)) return FALSE; From 81df5c8abae137bad8a08acefeab7b3a8e7205c1 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 15 Jul 2021 17:03:45 -0600 Subject: [PATCH 14/47] fixup! lib/repo: Add ostree_repo_remote_get_gpg_keys() --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d4c50440..0ff2c53f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2366,7 +2366,7 @@ _ostree_repo_gpg_prepare_verifier (OstreeRepo *self, /** * ostree_repo_remote_get_gpg_keys: * @self: an #OstreeRepo - * @name (nullable): name of the remote or %NULL + * @name: (nullable): name of the remote or %NULL * @key_ids: (array zero-terminated=1) (element-type utf8) (nullable): * a %NULL-terminated array of GPG key IDs to include, or %NULL * @out_keys: (out) (optional) (element-type GVariant) (transfer container): From 2c5fa2cdb6c304394ca629c959ec53d878f3f93a Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 26 Jul 2021 11:48:45 +0100 Subject: [PATCH 15/47] tests: Unset SOURCE_DATE_EPOCH Some distributions set this during build in order to have reproducible builds from the same source code: for example, Debian uses the date from debian/changelog. However, some of our tests assume that `ostree commit` will result in a commit with the current date/time, and SOURCE_DATE_EPOCH breaks that assumption. Unset it for our build-time tests. Resolves: https://github.com/ostreedev/ostree/issues/2405 Signed-off-by: Simon McVittie --- ci/gh-build.sh | 5 +++++ tests/libtest.sh | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/ci/gh-build.sh b/ci/gh-build.sh index 11434327..fdc79569 100755 --- a/ci/gh-build.sh +++ b/ci/gh-build.sh @@ -44,6 +44,11 @@ ${make} # Run the tests both using check and distcheck. ${make} check + +# Some tests historically failed when package builds set this. +# By setting it for distcheck but not check, we exercise both ways. +export SOURCE_DATE_EPOCH=$(date '+%s') + ${make} distcheck DISTCHECK_CONFIGURE_FLAGS="$*" # Show the installed files diff --git a/tests/libtest.sh b/tests/libtest.sh index 260b8d1d..f04ccaa0 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -69,6 +69,10 @@ if ! test -f .testtmp; then touch .testtmp fi +# Some distribution builds set this, but some of our build-time tests +# assume this won't be used when committing +unset SOURCE_DATE_EPOCH + # Also, unbreak `tar` inside `make check`...Automake will inject # TAR_OPTIONS: --owner=0 --group=0 --numeric-owner presumably so that # tarballs are predictable, except we don't want this in our tests. From 28174970c74903a856a87542a54c0bef1dacbc86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=BB=BA=E5=BC=BA?= Date: Mon, 26 Jul 2021 09:36:26 +0800 Subject: [PATCH 16/47] fix: Avoid wild pointers Pointer command is dangerous if there is no assignment. Log: Avoid wild pointers --- src/ostree/ot-main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index d153dcec..bbaba8b0 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -148,7 +148,6 @@ ostree_run (int argc, OstreeCommand *commands, GError **res_error) { - OstreeCommand *command; GError *error = NULL; GCancellable *cancellable = NULL; #ifndef BUILDOPT_TSAN @@ -187,7 +186,7 @@ ostree_run (int argc, argc = out; - command = commands; + OstreeCommand *command = commands; while (command->name) { if (g_strcmp0 (command_name, command->name) == 0) From 079528971ce9572e75d088b328efe5f84b52c988 Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Wed, 28 Jul 2021 18:30:33 -0400 Subject: [PATCH 17/47] workflows: bump lint toolchain --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 70233b8e..0dce242a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -8,7 +8,7 @@ on: env: CARGO_TERM_COLOR: always - ACTIONS_LINTS_TOOLCHAIN: 1.50.0 + ACTIONS_LINTS_TOOLCHAIN: 1.53.0 jobs: linting: From d9483f89ad2035b4cf4038b4ffe7dc297c2b7f0d Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Wed, 28 Jul 2021 18:32:04 -0400 Subject: [PATCH 18/47] workflows: limit permissions to reading repo contents Move the existing docs permissions stanza to the top of the workflow for consistency. --- .github/workflows/docs.yml | 10 ++++++---- .github/workflows/release.yml | 3 +++ .github/workflows/rust.yml | 4 ++++ .github/workflows/tests.yml | 4 ++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 05ede2e9..74f5e9d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,19 +1,21 @@ --- name: Docs + on: push: branches: [main] pull_request: branches: [main] +permissions: + # This workflow pushes to the gh-pages branch, so the token needs write + # privileges for repo contents. + contents: write + jobs: docs: name: Build documentation runs-on: ubuntu-latest - permissions: - # This job pushes to the gh-pages branch, so the token needs write - # privileges for repo contents. - contents: write steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 399b6637..e8fcd42b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,9 @@ on: paths: - 'configure.ac' +permissions: + contents: read + jobs: ci-release-build: name: "Sanity check release commits" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0dce242a..ef6e38a7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,11 +1,15 @@ --- name: Rust + on: push: branches: [main] pull_request: branches: [main] +permissions: + contents: read + env: CARGO_TERM_COLOR: always ACTIONS_LINTS_TOOLCHAIN: 1.53.0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45626be2..df1b1e07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,11 +1,15 @@ --- name: Tests + on: push: branches: [main] pull_request: branches: [main] +permissions: + contents: read + jobs: tests: # Distro configuration matrix From 75b17937cf69fbb215d3b18b6e6afba4dd9df82a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 3 Aug 2021 16:33:28 -0400 Subject: [PATCH 19/47] lib/sign-dummy: Handle incorrect signatures correctly We need to check all signatures for one which passes, not just fail on the first one. Reported-by: Seth Arnold --- src/libostree/ostree-sign-dummy.c | 7 ++++--- tests/test-delta-sign.sh | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 56f10d6e..55f28f11 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -171,7 +171,8 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_DUMMY_TYPE)) return glnx_throw (error, "signature: dummy: wrong type passed for verification"); - for (gsize i = 0; i < g_variant_n_children(signatures); i++) + gsize n = g_variant_n_children(signatures); + for (gsize i = 0; i < n; i++) { g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); g_autoptr (GBytes) signature = g_variant_get_data_as_bytes(child); @@ -188,9 +189,9 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, *out_success_message = g_strdup ("dummy: Signature verified"); return TRUE; } - else - return glnx_throw (error, "signature: dummy: incorrect signature %" G_GSIZE_FORMAT, i); } + if (n) + return glnx_throw (error, "signature: dummy: incorrect signatures found: %" G_GSIZE_FORMAT, n); return glnx_throw (error, "signature: dummy: no signatures"); } diff --git a/tests/test-delta-sign.sh b/tests/test-delta-sign.sh index 86f12f96..ed471db9 100755 --- a/tests/test-delta-sign.sh +++ b/tests/test-delta-sign.sh @@ -169,6 +169,6 @@ ostree_repo_init repo2 --mode=bare-user ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${origrev} ${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null ${CMD_PREFIX} ostree --repo=repo2 static-delta apply-offline --sign-type=dummy repo/deltas/${deltaprefix}/${deltadir} badsign 2> apply-offline-bad-key.txt && exit 1 -assert_file_has_content apply-offline-bad-key.txt "signature: dummy: incorrect signature" +assert_file_has_content apply-offline-bad-key.txt "signature: dummy: incorrect signatures found: 1" echo 'ok apply offline failed with dummy and bad key' From 738831c50b8a10383bf945283755120f4d68292d Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 3 Aug 2021 16:34:11 -0400 Subject: [PATCH 20/47] lib/sysroot: Fix error message about creating `/var/lib` Reported-by: Seth Arnold --- src/libostree/ostree-sysroot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 2fcd4323..be5306b7 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1790,7 +1790,7 @@ ostree_sysroot_init_osname (OstreeSysroot *self, return glnx_throw_errno_prefix (error, "fchmod %s", "var/tmp"); if (mkdirat (dfd, "var/lib", 0777) < 0) - return glnx_throw_errno_prefix (error, "Creating %s", "var/tmp"); + return glnx_throw_errno_prefix (error, "Creating %s", "var/lib"); /* This needs to be available and properly labeled early during the boot * process (before tmpfiles.d kicks in), so that journald can flush logs from From 0f95e4e5eeff2da8b89157a24a550f0a9e3fc382 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 3 Aug 2021 16:34:32 -0400 Subject: [PATCH 21/47] ostree/dump: Fix free'ing a static string Reported-by: Seth Arnold --- src/ostree/ot-dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index a8ed54a2..ac1a2b34 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -136,8 +136,8 @@ dump_commit (GVariant *variant, g_print ("Parent: %s\n", parent); } - g_autofree char *contents = ostree_commit_get_content_checksum (variant) ?: ""; - g_print ("ContentChecksum: %s\n", contents); + g_autofree char *contents = ostree_commit_get_content_checksum (variant); + g_print ("ContentChecksum: %s\n", contents ?: ""); g_print ("Date: %s\n", str); if ((version = ot_admin_checksum_version (variant))) From 5a3d5fb86f3d617c0c759c946fff92eec3e2bbad Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Tue, 17 Aug 2021 10:30:06 +0000 Subject: [PATCH 22/47] builtins/commit: check for conflicting permissions options This explicitly checks for commit command options asking for both non-zero UID/GID and canonical permissions at the same time, which are incompatible. --- src/ostree/ot-builtin-commit.c | 22 +++++++++++++++++----- tests/basic-test.sh | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 7a23741e..5aecdbdb 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -555,6 +555,23 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio detached_metadata = g_variant_ref_sink (g_variant_builder_end (builder)); } + /* Check for conflicting options */ + if (opt_canonical_permissions && opt_owner_uid > 0) + { + glnx_throw (error, "Cannot specify both --canonical-permissions and non-zero --owner-uid"); + goto out; + } + if (opt_canonical_permissions && opt_owner_gid > 0) + { + glnx_throw (error, "Cannot specify both --canonical-permissions and non-zero --owner-gid"); + goto out; + } + if (opt_selinux_policy && opt_selinux_policy_from_base) + { + glnx_throw (error, "Cannot specify both --selinux-policy and --selinux-policy-from-base"); + goto out; + } + if (opt_no_xattrs) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS; if (opt_consume) @@ -570,11 +587,6 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES; if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); - if (opt_selinux_policy && opt_selinux_policy_from_base) - { - glnx_throw (error, "Cannot specify both --selinux-policy and --selinux-policy-from-base"); - goto out; - } if (flags != 0 || opt_owner_uid >= 0 diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 03f2cb1b..4d1356eb 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -437,7 +437,7 @@ echo "ok user checkout" $OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit" --tree=ref=test2 echo "ok commit from ref" -$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit with modifier" --tree=ref=test2 --owner-uid=`id -u` +$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit with modifier" --tree=ref=test2 --owner-uid=0 echo "ok commit from ref with modifier" $OSTREE commit ${COMMIT_ARGS} -b trees/test2 -s 'ref with / in it' --tree=ref=test2 From b079c11381aa2b297652753d346b12b8112ef542 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Wed, 18 Aug 2021 09:06:26 +0000 Subject: [PATCH 23/47] builtins/commit: move commit modifier to auto-cleanup This reduces the usage of goto cleanup logic by porting the commit modifier pointer to autoptr. --- src/ostree/ot-builtin-commit.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 7a23741e..2663bd1c 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -432,7 +432,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio g_autoptr(GHashTable) skip_list = NULL; OstreeRepoCommitModifierFlags flags = 0; g_autoptr(OstreeSePolicy) policy = NULL; - OstreeRepoCommitModifier *modifier = NULL; + g_autoptr(OstreeRepoCommitModifier) modifier = NULL; OstreeRepoTransactionStats stats; struct CommitFilterData filter_data = { 0, }; g_autofree char *commit_body = NULL; @@ -968,7 +968,5 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio out: if (repo) ostree_repo_abort_transaction (repo, cancellable, NULL); - if (modifier) - ostree_repo_commit_modifier_unref (modifier); return ret; } From c6b72f527bd836d810890dfc6c8efd0229157fc3 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Thu, 19 Aug 2021 13:50:21 +0000 Subject: [PATCH 24/47] lib/core/checksum: add flag to use canonical permissions This adds a new `OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS` checksumming flag, which is needed in bare-user-only mode to ignore local IDs. --- src/libostree/ostree-core.c | 8 ++++++++ src/libostree/ostree-core.h | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 6a7c2afa..aecaf31a 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -984,6 +984,9 @@ ostree_checksum_file_at (int dfd, g_autoptr(GFileInfo) file_info = _ostree_stbuf_to_gfileinfo (stbuf); + const gboolean canonicalize_perms = + ((flags & OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS) != 0); + g_autoptr(GInputStream) in = NULL; if (S_ISREG (stbuf->st_mode)) { @@ -991,6 +994,11 @@ ostree_checksum_file_at (int dfd, if (!glnx_openat_rdonly (dfd, path, FALSE, &fd, error)) return FALSE; in = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE); + if (canonicalize_perms) + { + g_file_info_set_attribute_uint32 (file_info, "unix::uid", 0); + g_file_info_set_attribute_uint32 (file_info, "unix::gid", 0); + } } else if (S_ISLNK (stbuf->st_mode)) { diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 3b903d5c..7dc1ffb7 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -460,12 +460,21 @@ gboolean ostree_break_hardlink (int dfd, /** * OstreeChecksumFlags: + * @OSTREE_CHECKSUM_FLAGS_NONE: Default checksumming without tweaks. + * (Since: 2017.13.) + * @OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS: Ignore xattrs when checksumming. + * (Since: 2017.13.) + * @OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS: Use canonical uid/gid/mode + * values, for bare-user-only mode. (Since: 2021.4.) + * + * Flags influencing checksumming logic. * * Since: 2017.13 */ typedef enum { OSTREE_CHECKSUM_FLAGS_NONE = 0, OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS = (1 << 0), + OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS = (1 << 1), } OstreeChecksumFlags; _OSTREE_PUBLIC From d41fcd17a7956fd96831a8fed35309d194c69f9d Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Thu, 19 Aug 2021 14:07:19 +0000 Subject: [PATCH 25/47] lib/repo/checkout: use canonical perms in bare-user-only mode This automatically enables canonical permissions for checkouts in bare-user-only mode. --- src/libostree/ostree-repo-checkout.c | 8 +++++++- tests/basic-test.sh | 8 ++++---- tests/test-basic-user-only.sh | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 00c6a773..eaa33a28 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -375,6 +375,9 @@ create_file_copy_from_input_at (OstreeRepo *repo, if (repo->disable_xattrs) flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS; + if (repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + flags |= OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS; + g_autofree char *actual_checksum = NULL; if (!ostree_checksum_file_at (destination_dfd, destination_name, &dest_stbuf, OSTREE_OBJECT_TYPE_FILE, @@ -526,7 +529,10 @@ checkout_file_hardlink (OstreeRepo *self, * */ OstreeChecksumFlags flags = 0; if (self->disable_xattrs) - flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS; + flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS; + + if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + flags |= OSTREE_CHECKSUM_FLAGS_CANONICAL_PERMISSIONS; g_autofree char *actual_checksum = NULL; if (!ostree_checksum_file_at (destination_dfd, destination_name, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 4d1356eb..89d35273 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -31,7 +31,7 @@ if is_bare_user_only_repo repo; then # In bare-user-only repos we can only represent files with uid/gid 0, no # xattrs and canonical permissions, so we need to commit them as such, or # we end up with repos that don't pass fsck - COMMIT_ARGS="--canonical-permissions" + COMMIT_ARGS="--canonical-permissions --no-xattrs" DIFF_ARGS="--owner-uid=0 --owner-gid=0 --no-xattrs" # Also, since we can't check out uid=0 files we need to check out in user mode CHECKOUT_U_ARG="-U" @@ -706,7 +706,7 @@ for x in $(seq 3); do # But they share the GPL echo 'this is the GPL' > pkg${x}/usr/share/licenses/COPYING ln -s COPYING pkg${x}/usr/share/licenses/LICENSE - $OSTREE commit -b union-identical-pkg${x} --tree=dir=pkg${x} + $OSTREE commit ${COMMIT_ARGS} -b union-identical-pkg${x} --tree=dir=pkg${x} done rm union-identical-test -rf for x in $(seq 3); do @@ -756,7 +756,7 @@ cd ${test_tmpdir} rm files -rf && mkdir files touch files/anemptyfile touch files/anotheremptyfile -$CMD_PREFIX ostree --repo=repo commit --consume -b tree-with-empty-files --tree=dir=files +$CMD_PREFIX ostree --repo=repo commit ${COMMIT_ARGS} --consume -b tree-with-empty-files --tree=dir=files $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} tree-with-empty-files tree-with-empty-files if files_are_hardlinked tree-with-empty-files/an{,other}emptyfile; then fatal "--force-copy-zerosized failed" @@ -773,7 +773,7 @@ echo "ok checkout zero sized files are not hardlinked" $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} --union-identical -z tree-with-empty-files tree-with-empty-files $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} --union-identical -z tree-with-empty-files tree-with-empty-files echo notempty > tree-with-empty-files/anemptyfile.new && mv tree-with-empty-files/anemptyfile{.new,} -$CMD_PREFIX ostree --repo=repo commit --consume -b tree-with-conflicting-empty-files --tree=dir=tree-with-empty-files +$CMD_PREFIX ostree --repo=repo commit ${COMMIT_ARGS} --consume -b tree-with-conflicting-empty-files --tree=dir=tree-with-empty-files # Reset back to base rm tree-with-empty-files -rf $CMD_PREFIX ostree --repo=repo checkout ${CHECKOUT_H_ARGS} --union-identical -z tree-with-empty-files tree-with-empty-files diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index f65094fd..02129b28 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -25,7 +25,7 @@ set -euo pipefail mode="bare-user-only" setup_test_repository "$mode" -extra_basic_tests=5 +extra_basic_tests=6 . $(dirname $0)/basic-test.sh $CMD_PREFIX ostree --version > version.yaml @@ -41,6 +41,7 @@ ostree_repo_init repo init --mode=bare-user-only cd ${test_tmpdir} rm repo-input -rf ostree_repo_init repo-input init --mode=archive + cd ${test_tmpdir} cat > statoverride.txt < files/afile +$OSTREE commit ${COMMIT_ARGS} -b perms files +rm out -rf +$OSTREE checkout --force-copy perms out +$OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical perms out +$OSTREE fsck +echo "ok checkout checksum with canonical perms" From 8a5241dd6a6bc4d30195c9d461f50248b0230fc5 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Fri, 20 Aug 2021 10:58:24 +0000 Subject: [PATCH 26/47] lib/commit: autofix permissions for bare-user-only This tweaks commit logic to detect bare-user-only repositories and canonicalize permissions automatically. --- src/libostree/ostree-repo-commit.c | 29 +++++++++++++++++++++-------- src/libostree/ostree-repo.h | 6 +++++- tests/basic-test.sh | 2 +- tests/test-basic-user-only.sh | 14 +++++++++++++- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index c07526fc..249e792c 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3286,22 +3286,35 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, GFileInfo *file_info, GFileInfo **out_modified_info) { + gboolean canonicalize_perms = FALSE; + gboolean has_filter = FALSE; OstreeRepoCommitFilterResult result = OSTREE_REPO_COMMIT_FILTER_ALLOW; GFileInfo *modified_info; - if (modifier == NULL || - (modifier->filter == NULL && - (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS) == 0)) + /* Auto-detect bare-user-only repo, force canonical permissions. */ + if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + canonicalize_perms = TRUE; + + if (modifier != NULL) + { + if ((modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS) != 0) + canonicalize_perms = TRUE; + if (modifier->filter != NULL) + has_filter = TRUE; + } + + if (!(canonicalize_perms || has_filter)) { *out_modified_info = g_object_ref (file_info); - return OSTREE_REPO_COMMIT_FILTER_ALLOW; + return OSTREE_REPO_COMMIT_FILTER_ALLOW; /* Note: early return (no actions needed) */ } modified_info = g_file_info_dup (file_info); - if (modifier->filter) + + if (has_filter) result = modifier->filter (self, path, modified_info, modifier->user_data); - if ((modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS) != 0) + if (canonicalize_perms) { guint mode = g_file_info_get_attribute_uint32 (modified_info, "unix::mode"); switch (g_file_info_get_file_type (file_info)) @@ -3618,8 +3631,8 @@ write_content_to_mtree_internal (OstreeRepo *self, /* Load flags into boolean constants for ease of readability (we also need to * NULL-check modifier) */ - const gboolean canonical_permissions = modifier && - (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS); + const gboolean canonical_permissions = self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY || + (modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS)); const gboolean devino_canonical = modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL); /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 08d3d408..1a9aa325 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -678,10 +678,14 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo *r * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_NONE: No special flags * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS: Do not process extended attributes * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES: Generate size information. - * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS: Canonicalize permissions for bare-user-only mode. + * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS: Canonicalize permissions. * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_ERROR_ON_UNLABELED: Emit an error if configured SELinux policy does not provide a label * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME: Delete added files/directories after commit; Since: 2017.13 * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL: If a devino cache hit is found, skip modifier filters (non-directories only); Since: 2017.14 + * + * Flags modifying commit behavior. In bare-user-only mode, @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS + * is automatically enabled. + * */ typedef enum { OSTREE_REPO_COMMIT_MODIFIER_FLAGS_NONE = 0, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 89d35273..7946ffa3 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -31,7 +31,7 @@ if is_bare_user_only_repo repo; then # In bare-user-only repos we can only represent files with uid/gid 0, no # xattrs and canonical permissions, so we need to commit them as such, or # we end up with repos that don't pass fsck - COMMIT_ARGS="--canonical-permissions --no-xattrs" + COMMIT_ARGS="--no-xattrs" DIFF_ARGS="--owner-uid=0 --owner-gid=0 --no-xattrs" # Also, since we can't check out uid=0 files we need to check out in user mode CHECKOUT_U_ARG="-U" diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index 02129b28..7ef153c3 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -25,7 +25,7 @@ set -euo pipefail mode="bare-user-only" setup_test_repository "$mode" -extra_basic_tests=6 +extra_basic_tests=7 . $(dirname $0)/basic-test.sh $CMD_PREFIX ostree --version > version.yaml @@ -112,3 +112,15 @@ $OSTREE checkout --force-copy perms out $OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical perms out $OSTREE fsck echo "ok checkout checksum with canonical perms" + +cd ${test_tmpdir} +rm repo -rf +ostree_repo_init repo init --mode=bare-user-only +rm files -rf && mkdir files +echo afile > files/afile +$OSTREE commit ${COMMIT_ARGS} -b perms files +rm out -rf +$OSTREE checkout --force-copy perms out +$OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical perms out +$OSTREE fsck +echo "ok automatic canonical perms for bare-user-only" From 06ff77cfeb7eef5e75528f787f8c5569cd5b2334 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 23 Aug 2021 12:39:02 +0000 Subject: [PATCH 27/47] lib/diff: ignore xattrs if disabled on either repos This fixes the logic to detect whether xattrs should be automatically ignored when diffing. --- src/libostree/ostree-diff.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index da018db4..e2d68d0a 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -268,15 +268,18 @@ ostree_diff_dirs_with_options (OstreeDiffFlags flags, /* If we're diffing versus a repo, and either of them have xattrs disabled, * then disable for both. */ - OstreeRepo *repo; if (OSTREE_IS_REPO_FILE (a)) - repo = ostree_repo_file_get_repo ((OstreeRepoFile*)a); - else if (OSTREE_IS_REPO_FILE (b)) - repo = ostree_repo_file_get_repo ((OstreeRepoFile*)b); - else - repo = NULL; - if (repo != NULL && repo->disable_xattrs) - flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + { + OstreeRepo *repo = ostree_repo_file_get_repo ((OstreeRepoFile*)a); + if (repo->disable_xattrs) + flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + } + if (OSTREE_IS_REPO_FILE (b)) + { + OstreeRepo *repo = ostree_repo_file_get_repo ((OstreeRepoFile*)b); + if (repo->disable_xattrs) + flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + } if (a == NULL) { From 58a683f8f0a14d19d15e3b5521b9fbcc6c04a9a1 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 23 Aug 2021 11:09:24 -0600 Subject: [PATCH 28/47] bin/remote: Rename list-gpg-keys to gpg-list-keys As pointed out in the original review, `gpg-list-keys` fits better alongside the existing `gpg-import`. Changes were done with: ``` git grep -l list-gpg-keys | xargs sed -i 's/list-gpg-keys/gpg-list-keys/' for src in $(git ls-files '*list-gpg-keys*'); do dst=${src/list-gpg-keys/gpg-list-keys} git mv "$src" "$dst" done ``` --- Makefile-ostree.am | 2 +- Makefile-tests.am | 2 +- bash/ostree | 2 +- man/ostree-remote.xml | 4 ++-- src/ostree/ot-builtin-remote.c | 2 +- ...pg-keys.c => ot-remote-builtin-gpg-list-keys.c} | 0 ...st-gpg-keys.sh => test-remote-gpg-list-keys.sh} | 14 +++++++------- 7 files changed, 13 insertions(+), 13 deletions(-) rename src/ostree/{ot-remote-builtin-list-gpg-keys.c => ot-remote-builtin-gpg-list-keys.c} (100%) rename tests/{test-remote-list-gpg-keys.sh => test-remote-gpg-list-keys.sh} (94%) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index a5509f7c..dde10c17 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -105,7 +105,7 @@ ostree_SOURCES += \ if USE_GPGME ostree_SOURCES += \ src/ostree/ot-remote-builtin-gpg-import.c \ - src/ostree/ot-remote-builtin-list-gpg-keys.c \ + src/ostree/ot-remote-builtin-gpg-list-keys.c \ $(NULL) endif diff --git a/Makefile-tests.am b/Makefile-tests.am index 1997bfd8..81fe2b76 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -152,7 +152,7 @@ _installed_or_uninstalled_test_scripts = \ if USE_GPGME _installed_or_uninstalled_test_scripts += \ tests/test-remote-gpg-import.sh \ - tests/test-remote-list-gpg-keys.sh \ + tests/test-remote-gpg-list-keys.sh \ tests/test-gpg-signed-commit.sh \ tests/test-admin-gpg.sh \ $(NULL) diff --git a/bash/ostree b/bash/ostree index 32d5e317..c990462f 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1381,9 +1381,9 @@ _ostree_remote() { delete delete-cookie gpg-import + gpg-list-keys list list-cookies - list-gpg-keys refs show-url summary diff --git a/man/ostree-remote.xml b/man/ostree-remote.xml index 928bf9b5..20fe0a19 100644 --- a/man/ostree-remote.xml +++ b/man/ostree-remote.xml @@ -66,7 +66,7 @@ Boston, MA 02111-1307, USA. ostree remote gpg-import OPTIONS NAME KEY-ID - ostree remote list-gpg-keys NAME + ostree remote gpg-list-keys NAME ostree remote refs NAME @@ -112,7 +112,7 @@ Boston, MA 02111-1307, USA. The gpg-import subcommand can associate GPG keys to a specific remote repository for use when pulling signed commits from that repository (if GPG verification is enabled). The - list-gpg-keys subcommand can be used to see the + gpg-list-keys subcommand can be used to see the GPG keys currently associated with a remote repository. diff --git a/src/ostree/ot-builtin-remote.c b/src/ostree/ot-builtin-remote.c index 7028eacc..3c0d9d2e 100644 --- a/src/ostree/ot-builtin-remote.c +++ b/src/ostree/ot-builtin-remote.c @@ -44,7 +44,7 @@ static OstreeCommand remote_subcommands[] = { { "gpg-import", OSTREE_BUILTIN_FLAG_NONE, ot_remote_builtin_gpg_import, "Import GPG keys" }, - { "list-gpg-keys", OSTREE_BUILTIN_FLAG_NONE, + { "gpg-list-keys", OSTREE_BUILTIN_FLAG_NONE, ot_remote_builtin_list_gpg_keys, "Show remote GPG keys" }, #endif /* OSTREE_DISABLE_GPGME */ diff --git a/src/ostree/ot-remote-builtin-list-gpg-keys.c b/src/ostree/ot-remote-builtin-gpg-list-keys.c similarity index 100% rename from src/ostree/ot-remote-builtin-list-gpg-keys.c rename to src/ostree/ot-remote-builtin-gpg-list-keys.c diff --git a/tests/test-remote-list-gpg-keys.sh b/tests/test-remote-gpg-list-keys.sh similarity index 94% rename from tests/test-remote-list-gpg-keys.sh rename to tests/test-remote-gpg-list-keys.sh index 81699f14..51b60084 100755 --- a/tests/test-remote-list-gpg-keys.sh +++ b/tests/test-remote-gpg-list-keys.sh @@ -46,7 +46,7 @@ cd ${test_tmpdir} ${OSTREE} remote add R1 http://example.com/repo # No remote keyring should list no keys. -${OSTREE} remote list-gpg-keys R1 > result +${OSTREE} remote gpg-list-keys R1 > result assert_file_empty result echo "ok remote no keyring" @@ -54,7 +54,7 @@ echo "ok remote no keyring" # Make the global keyring available and make sure there are still no # keys found for a specified remote. OSTREE_GPG_HOME=${trusteddir} -${OSTREE} remote list-gpg-keys R1 > result +${OSTREE} remote gpg-list-keys R1 > result OSTREE_GPG_HOME=${emptydir} assert_file_empty result @@ -62,7 +62,7 @@ echo "ok remote with global keyring" # Import a key and check that it's listed ${OSTREE} remote gpg-import --keyring ${TEST_GPG_KEYHOME}/key1.asc R1 -${OSTREE} remote list-gpg-keys R1 > result +${OSTREE} remote gpg-list-keys R1 > result cat > expected <<"EOF" Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA Created: Tue Sep 10 02:29:42 2013 @@ -78,14 +78,14 @@ echo "ok remote with keyring" # Check the global keys with no keyring OSTREE_GPG_HOME=${emptydir} -${OSTREE} remote list-gpg-keys > result +${OSTREE} remote gpg-list-keys > result assert_file_empty result echo "ok global no keyring" # Now check the global keys with a keyring OSTREE_GPG_HOME=${trusteddir} -${OSTREE} remote list-gpg-keys > result +${OSTREE} remote gpg-list-keys > result OSTREE_GPG_HOME=${emptydir} cat > expected <<"EOF" Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA @@ -134,7 +134,7 @@ else sleep 2 ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1expired.asc ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1expired.asc R1 - ${OSTREE} remote list-gpg-keys R1 > result + ${OSTREE} remote gpg-list-keys R1 > result assert_file_has_content result "^ Expired:" echo "ok remote expired key" @@ -143,7 +143,7 @@ else ${GPG} --homedir=${TEST_GPG_KEYHOME} --import ${TEST_GPG_KEYHOME}/revocations/key1.rev ${GPG} --homedir=${test_tmpdir}/gpghome --armor --export ${TEST_GPG_KEYID_1} > ${test_tmpdir}/key1revoked.asc ${OSTREE} remote gpg-import --keyring ${test_tmpdir}/key1revoked.asc R1 - ${OSTREE} remote list-gpg-keys R1 > result + ${OSTREE} remote gpg-list-keys R1 > result assert_file_has_content result "^Key: 5E65DE75AB1C501862D476347FCA23D8472CDAFA (revoked)" assert_file_has_content result "^ UID: Ostree Tester (revoked)" assert_file_has_content result "^ Subkey: CC47B2DFB520AEF231180725DF20F58B408DEA49 (revoked)" From 0276f4dc0c02170b763a796f5811adc2e7dd80ed Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 23 Aug 2021 13:51:55 +0000 Subject: [PATCH 29/47] lib/diff: automatically skip xattrs in bare-user-only mode --- src/libostree/ostree-diff.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index e2d68d0a..a5c02b7b 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -271,13 +271,13 @@ ostree_diff_dirs_with_options (OstreeDiffFlags flags, if (OSTREE_IS_REPO_FILE (a)) { OstreeRepo *repo = ostree_repo_file_get_repo ((OstreeRepoFile*)a); - if (repo->disable_xattrs) + if (repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; } if (OSTREE_IS_REPO_FILE (b)) { OstreeRepo *repo = ostree_repo_file_get_repo ((OstreeRepoFile*)b); - if (repo->disable_xattrs) + if (repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; } From f75552e15ca1afb3ed05b3d16abfcb1e9de3c57d Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 23 Aug 2021 13:08:53 +0000 Subject: [PATCH 30/47] builtins/commit: set up relevant flags in bare-user-only mode This detects bare-user-only mode and automatically enables a commit modifier with relevant flags. --- src/ostree/ot-builtin-commit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 3698a2f4..370e085c 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -572,7 +572,9 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio goto out; } - if (opt_no_xattrs) + if (opt_canonical_permissions || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS; + if (opt_no_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS; if (opt_consume) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME; @@ -581,8 +583,6 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio opt_link_checkout_speedup = TRUE; /* Imply this */ flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL; } - if (opt_canonical_permissions) - flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS; if (opt_generate_sizes) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES; if (opt_disable_fsync) From 3e2360e3bba6c215acccc77373d01a5674c770e6 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 23 Aug 2021 13:08:44 +0000 Subject: [PATCH 31/47] lib/commit: automatically skip xattrs in bare-user-only mode --- src/libostree/ostree-repo-commit.c | 5 +++-- src/libostree/ostree-repo.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 249e792c..d5ab57a2 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3382,8 +3382,9 @@ get_final_xattrs (OstreeRepo *self, /* track whether the returned xattrs differ from the file on disk */ gboolean modified = TRUE; const gboolean skip_xattrs = (modifier && - modifier->flags & (OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS | - OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS)) > 0; + (modifier->flags & (OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS | + OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS)) > 0) || + self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY; /* fetch on-disk xattrs if needed & not disabled */ g_autoptr(GVariant) original_xattrs = NULL; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index bbec1a37..962fa8cc 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -684,7 +684,7 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo *r * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL: If a devino cache hit is found, skip modifier filters (non-directories only); Since: 2017.14 * * Flags modifying commit behavior. In bare-user-only mode, @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS - * is automatically enabled. + * and @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS are automatically enabled. * */ typedef enum { From 00660eae7972a04ef9a0784e504545f43376c517 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 23 Aug 2021 09:46:22 +0000 Subject: [PATCH 32/47] tests: update several bare-user-only checks --- tests/basic-test.sh | 19 ++++++++++--------- tests/test-basic-user-only.sh | 27 +++++++++------------------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 7946ffa3..935544d9 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -29,10 +29,8 @@ COMMIT_ARGS="" DIFF_ARGS="" if is_bare_user_only_repo repo; then # In bare-user-only repos we can only represent files with uid/gid 0, no - # xattrs and canonical permissions, so we need to commit them as such, or - # we end up with repos that don't pass fsck - COMMIT_ARGS="--no-xattrs" - DIFF_ARGS="--owner-uid=0 --owner-gid=0 --no-xattrs" + # xattrs and canonical permissions. + DIFF_ARGS="--owner-uid=0 --owner-gid=0" # Also, since we can't check out uid=0 files we need to check out in user mode CHECKOUT_U_ARG="-U" CHECKOUT_H_ARGS="-U -H" @@ -314,7 +312,7 @@ echo "ok diff revisions" cd ${test_tmpdir}/checkout-test2-4 echo afile > oh-look-a-file -$OSTREE diff test2 ./ > ${test_tmpdir}/diff-test2-2 +$OSTREE diff ${DIFF_ARGS} test2 ./ > ${test_tmpdir}/diff-test2-2 rm oh-look-a-file cd ${test_tmpdir} assert_file_has_content diff-test2-2 'A *oh-look-a-file$' @@ -787,11 +785,14 @@ cd ${test_tmpdir} rm files -rf && mkdir files mkdir files/worldwritable-dir chmod a+w files/worldwritable-dir -$CMD_PREFIX ostree --repo=repo commit -b content-with-dir-world-writable --tree=dir=files +$OSTREE commit ${COMMIT_ARGS} -b content-with-dir-world-writable --tree=dir=files +$OSTREE fsck rm dir-co -rf -$CMD_PREFIX ostree --repo=repo checkout -U -H -M content-with-dir-world-writable dir-co -assert_file_has_mode dir-co/worldwritable-dir 775 -if ! is_bare_user_only_repo repo; then +$OSTREE checkout -U -H -M content-with-dir-world-writable dir-co +if is_bare_user_only_repo repo; then + assert_file_has_mode dir-co/worldwritable-dir 755 +else + assert_file_has_mode dir-co/worldwritable-dir 775 rm dir-co -rf $CMD_PREFIX ostree --repo=repo checkout -U -H content-with-dir-world-writable dir-co assert_file_has_mode dir-co/worldwritable-dir 777 diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index 7ef153c3..568f9e95 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -25,7 +25,7 @@ set -euo pipefail mode="bare-user-only" setup_test_repository "$mode" -extra_basic_tests=7 +extra_basic_tests=6 . $(dirname $0)/basic-test.sh $CMD_PREFIX ostree --version > version.yaml @@ -63,8 +63,8 @@ rm files -rf && mkdir files echo "a group writable file" > files/some-group-writable chmod 0664 files/some-group-writable $CMD_PREFIX ostree --repo=repo-input commit -b content-with-group-writable --tree=dir=files -$CMD_PREFIX ostree pull-local --repo=repo repo-input -$CMD_PREFIX ostree --repo=repo checkout -U -H content-with-group-writable groupwritable-co +$OSTREE pull-local repo-input +$OSTREE checkout -U -H content-with-group-writable groupwritable-co assert_file_has_mode groupwritable-co/some-group-writable 664 echo "ok supported group writable" @@ -75,8 +75,8 @@ rm files -rf && mkdir files mkdir files/worldwritable-dir chmod a+w files/worldwritable-dir $CMD_PREFIX ostree --repo=repo-input commit -b content-with-dir-world-writable --tree=dir=files -$CMD_PREFIX ostree pull-local --repo=repo repo-input -$CMD_PREFIX ostree --repo=repo checkout -U -H content-with-dir-world-writable dir-co +$OSTREE pull-local repo-input +$OSTREE checkout -U -H content-with-dir-world-writable dir-co assert_file_has_mode dir-co/worldwritable-dir 775 echo "ok didn't make world-writable dir" @@ -106,21 +106,12 @@ rm repo -rf ostree_repo_init repo init --mode=bare-user-only rm files -rf && mkdir files echo afile > files/afile +chmod 0777 files/afile $OSTREE commit ${COMMIT_ARGS} -b perms files +$OSTREE fsck rm out -rf $OSTREE checkout --force-copy perms out +assert_file_has_mode out/afile 755 $OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical perms out -$OSTREE fsck -echo "ok checkout checksum with canonical perms" - -cd ${test_tmpdir} -rm repo -rf -ostree_repo_init repo init --mode=bare-user-only -rm files -rf && mkdir files -echo afile > files/afile -$OSTREE commit ${COMMIT_ARGS} -b perms files -rm out -rf -$OSTREE checkout --force-copy perms out -$OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical perms out -$OSTREE fsck +assert_file_has_mode out/afile 755 echo "ok automatic canonical perms for bare-user-only" From c64b4bcebafd06a0cbfc4b8e67fb3e20e99488bb Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Wed, 25 Aug 2021 12:30:21 +0000 Subject: [PATCH 33/47] lib: improve transactions auto-cleanup logic This fixes some aspects of OstreeRepoAutoTransaction and re-aligns it with the logic in flatpak. Specifically: * link to the underlying repo through refcounting * bridge internal errors to warning messages * verify the input pointer type This is a preparation step before exposing this logic as a public API. --- src/libostree/ostree-repo-private.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 0cd9d8bb..67f755bd 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -234,9 +234,17 @@ typedef OstreeRepo _OstreeRepoAutoTransaction; static inline void _ostree_repo_auto_transaction_cleanup (void *p) { + if (p == NULL) + return; + g_return_if_fail (OSTREE_IS_REPO (p)); + OstreeRepo *repo = p; - if (repo) - (void) ostree_repo_abort_transaction (repo, NULL, NULL); + g_autoptr(GError) error = NULL; + + if (!ostree_repo_abort_transaction (repo, NULL, &error)) + g_warning("Failed to auto-cleanup OSTree transaction: %s", error->message); + + g_object_unref (repo); } static inline _OstreeRepoAutoTransaction * @@ -246,7 +254,8 @@ _ostree_repo_auto_transaction_start (OstreeRepo *repo, { if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error)) return NULL; - return (_OstreeRepoAutoTransaction *)repo; + + return (_OstreeRepoAutoTransaction *) g_object_ref (repo); } G_DEFINE_AUTOPTR_CLEANUP_FUNC (_OstreeRepoAutoTransaction, _ostree_repo_auto_transaction_cleanup) From 3e30e72d337109eb64ae662ef4ebacd5453d3268 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 25 Aug 2021 15:18:43 -0400 Subject: [PATCH 34/47] lib: Change read_commit_detached_metadata to be nullable Hit this while working on some Rust code. --- src/libostree/ostree-repo-commit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index d5ab57a2..8dc2355e 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3117,7 +3117,7 @@ ostree_repo_write_commit_with_time (OstreeRepo *self, * ostree_repo_read_commit_detached_metadata: * @self: Repo * @checksum: ASCII SHA256 commit checksum - * @out_metadata: (out) (transfer full): Metadata associated with commit in with format "a{sv}", or %NULL if none exists + * @out_metadata: (out) (nullable) (transfer full): Metadata associated with commit in with format "a{sv}", or %NULL if none exists * @cancellable: Cancellable * @error: Error * @@ -3132,6 +3132,8 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error) { + g_assert (out_metadata != NULL); + char buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode); From fdeee165f6f82b2902fb791e33472e4600845a2b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 26 Aug 2021 09:53:56 -0400 Subject: [PATCH 35/47] ci: Run main GH action CI build+test as non-root This is really the standard best practice, matching how e.g. dpkg/rpm work, as well as most local development environments (including mine) with e.g. `toolbox`. --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index df1b1e07..c17a1c0d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -120,8 +120,11 @@ jobs: - name: Install dependencies run: ./ci/gh-install.sh ${{ matrix.extra-packages }} + - name: Add non-root user + run: "useradd builder && chown -R -h builder: ." + - name: Build and test - run: ./ci/gh-build.sh ${{ matrix.configure-options }} + run: runuser -u builder -- ./ci/gh-build.sh ${{ matrix.configure-options }} env: # GitHub hosted runners currently have 2 CPUs, so run 2 # parallel make jobs. From 87d115706eed52069e54785b87108c97a44a0343 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 26 Aug 2021 15:16:37 -0400 Subject: [PATCH 36/47] checkout: Save errno when re-throwing I was seeing an `EPERM` here which was confusing. It turned out the real error was `EEXIST`. Since we're referring to the original error, but we do a lot of computation in the middle, we need to save errno. --- src/libostree/ostree-repo-checkout.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index eaa33a28..bdd93e7c 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -481,6 +481,7 @@ checkout_file_hardlink (OstreeRepo *self, } else if (errno == EEXIST) { + int saved_errno = errno; /* When we get EEXIST, we need to handle the different overwrite modes. */ switch (options->overwrite_mode) { @@ -566,6 +567,7 @@ checkout_file_hardlink (OstreeRepo *self, else { g_assert_cmpint (options->overwrite_mode, ==, OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL); + errno = saved_errno; return glnx_throw_errno_prefix (error, "Hardlinking %s to %s", loose_path, destination_name); } break; From dd506fe2c8fea6bef0cd9533ac7f978eaff6f4a8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 26 Aug 2021 15:25:52 -0400 Subject: [PATCH 37/47] checkout: Also ignore xattrs for union in bare-user-only mode Followup to PRs related to https://github.com/ostreedev/ostree/issues/2410 Since the test suite now covers this the test was failing on a Fedora SELinux enabled host where we see `security.selinux` even if not in the commit. --- src/libostree/ostree-repo-checkout.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index eaa33a28..2dec8545 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -372,7 +372,7 @@ create_file_copy_from_input_at (OstreeRepo *repo, * checkout_file_hardlink(). */ OstreeChecksumFlags flags = 0; - if (repo->disable_xattrs) + if (repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS; if (repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) @@ -528,7 +528,7 @@ checkout_file_hardlink (OstreeRepo *self, * shouldn't hit this anymore. https://github.com/ostreedev/ostree/pull/1258 * */ OstreeChecksumFlags flags = 0; - if (self->disable_xattrs) + if (self->disable_xattrs || self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS; if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) From 359435de843ce2a1e94941c24ec4ddd7d5a7bccb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Apr 2021 18:42:05 -0400 Subject: [PATCH 38/47] Add an API to verify a commit signature explicitly We have a bunch of APIs to do GPG verification of a commit, but that doesn't generalize to signapi. Further, they require the caller to check the signature status explicitly which seems like a trap. This much higher level API works with both GPG and signapi. The intention is to use this in things that are doing "external pulls" like the ostree-ext tar import support. There we will get the commitmeta from the tarball and we want to verify it at the same time we import the commit. --- Makefile-tests.am | 4 + apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-pull-verify.c | 115 +++++++++++++++++++++++ src/libostree/ostree-repo.h | 23 +++++ src/ostree/ot-admin-builtin-status.c | 36 +++++++- tests/.gitignore | 1 + tests/test-admin-gpg.sh | 5 + tests/test-commit-sign-sh-ext.c | 118 ++++++++++++++++++++++++ tests/test-commit-sign.sh | 10 +- 10 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 tests/test-commit-sign-sh-ext.c diff --git a/Makefile-tests.am b/Makefile-tests.am index 81fe2b76..efbcad9a 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -391,6 +391,10 @@ tests_test_rfc2616_dates_SOURCES = \ tests_test_rfc2616_dates_CFLAGS = $(TESTS_CFLAGS) tests_test_rfc2616_dates_LDADD = $(TESTS_LDADD) +noinst_PROGRAMS += tests/test-commit-sign-sh-ext +tests_test_commit_sign_sh_ext_CFLAGS = $(TESTS_CFLAGS) +tests_test_commit_sign_sh_ext_LDADD = $(TESTS_LDADD) + if USE_GPGME tests_test_gpg_verify_result_SOURCES = \ src/libostree/ostree-gpg-verify-result-private.h \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 4d027555..f0901f21 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -474,6 +474,7 @@ ostree_repo_append_gpg_signature ostree_repo_add_gpg_signature_summary ostree_repo_gpg_sign_data ostree_repo_gpg_verify_data +ostree_repo_signature_verify_commit_data ostree_repo_verify_commit ostree_repo_verify_commit_ext ostree_repo_verify_commit_for_remote diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 75bc4647..7e6f7784 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -25,6 +25,7 @@ LIBOSTREE_2021.4 { global: ostree_repo_remote_get_gpg_keys; + ostree_repo_signature_verify_commit_data; } LIBOSTREE_2021.3; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c index fa170f94..e469dc0b 100644 --- a/src/libostree/ostree-repo-pull-verify.c +++ b/src/libostree/ostree-repo-pull-verify.c @@ -270,6 +270,7 @@ _sign_verify_for_remote (GPtrArray *verifiers, g_assert (out_success_message == NULL || *out_success_message == NULL); + g_assert (verifiers); g_assert_cmpuint (verifiers->len, >=, 1); for (guint i = 0; i < verifiers->len; i++) { @@ -346,6 +347,120 @@ _process_gpg_verify_result (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ +static gboolean +validate_metadata_size (const char *prefix, GBytes *buf, GError **error) +{ + gsize len = g_bytes_get_size (buf); + if (len > OSTREE_MAX_METADATA_SIZE) + return glnx_throw (error, "%s is %" G_GUINT64_FORMAT " bytes, exceeding maximum %" G_GUINT64_FORMAT, prefix, (guint64)len, (guint64)OSTREE_MAX_METADATA_SIZE); + return TRUE; +} + +/** + * ostree_repo_signature_verify_commit_data: + * @self: Repo + * @remote_name: Name of remote + * @commit_data: Commit object data (GVariant) + * @commit_metadata: Commit metadata (GVariant `a{sv}`), must contain at least one valid signature + * @flags: Optionally disable GPG or signapi + * @out_results: (nullable) (out) (transfer full): Textual description of results + * @error: Error + * + * Validate the commit data using the commit metadata which must + * contain at least one valid signature. If GPG and signapi are + * both enabled, then both must find at least one valid signature. + */ +gboolean +ostree_repo_signature_verify_commit_data (OstreeRepo *self, + const char *remote_name, + GBytes *commit_data, + GBytes *commit_metadata, + OstreeRepoVerifyFlags flags, + char **out_results, + GError **error) +{ + g_assert (self); + g_assert (remote_name); + g_assert (commit_data); + + gboolean gpg = !(flags & OSTREE_REPO_VERIFY_FLAGS_NO_GPG); + gboolean signapi = !(flags & OSTREE_REPO_VERIFY_FLAGS_NO_SIGNAPI); + // Must ask for at least one type of verification + if (!(gpg || signapi)) + return glnx_throw (error, "No commit verification types enabled via API"); + + if (!validate_metadata_size ("Commit", commit_data, error)) + return FALSE; + /* Nothing to check if detached metadata is absent */ + if (commit_metadata == NULL) + return glnx_throw (error, "Can't verify commit without detached metadata"); + if (!validate_metadata_size ("Commit metadata", commit_metadata, error)) + return FALSE; + g_autoptr(GVariant) commit_metadata_v = g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, commit_metadata, FALSE); + + g_autoptr(GString) results_buf = g_string_new (""); + gboolean verified = FALSE; + + if (gpg) + { + if (!ostree_repo_remote_get_gpg_verify (self, remote_name, + &gpg, error)) + return FALSE; + } + + /* TODO - we could cache this in the repo */ + g_autoptr(GPtrArray) signapi_verifiers = NULL; + if (signapi) + { + if (!_signapi_init_for_remote (self, remote_name, &signapi_verifiers, NULL, error)) + return FALSE; + } + + if (!(gpg || signapi_verifiers)) + return glnx_throw (error, "Cannot verify commit for remote %s; GPG verification disabled, and no signapi verifiers configured", remote_name); + +#ifndef OSTREE_DISABLE_GPGME + if (gpg) + { + g_autoptr(OstreeGpgVerifyResult) result = + _ostree_repo_gpg_verify_with_metadata (self, commit_data, + commit_metadata_v, + remote_name, + NULL, NULL, NULL, error); + if (!result) + return FALSE; + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + return FALSE; + + const guint n_signatures = ostree_gpg_verify_result_count_all (result); + g_assert_cmpuint (n_signatures, >, 0); + for (guint jj = 0; jj < n_signatures; jj++) + { + ostree_gpg_verify_result_describe (result, jj, results_buf, "GPG: ", + OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT); + } + verified = TRUE; + } +#endif /* OSTREE_DISABLE_GPGME */ + + if (signapi_verifiers) + { + g_autofree char *success_message = NULL; + if (!_sign_verify_for_remote (signapi_verifiers, commit_data, commit_metadata_v, &success_message, error)) + return glnx_prefix_error (error, "Can't verify commit"); + if (verified) + g_string_append_c (results_buf, '\n'); + g_string_append (results_buf, success_message); + verified = TRUE; + } + + /* Must be true since we did g_assert (gpg || signapi) */ + g_assert (verified); + if (out_results) + *out_results = g_string_free (g_steal_pointer (&results_buf), FALSE); + return TRUE; +} + gboolean _verify_unwritten_commit (OtPullData *pull_data, const char *checksum, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 962fa8cc..522cb034 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1538,6 +1538,29 @@ OstreeGpgVerifyResult * ostree_repo_verify_summary (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeRepoVerifyFlags: + * @OSTREE_REPO_VERIFY_FLAGS_NONE: No flags + * @OSTREE_REPO_VERIFY_FLAGS_NO_GPG: Skip GPG verification + * @OSTREE_REPO_VERIFY_FLAGS_NO_SIGNAPI: Skip all other signature verification methods + * + * Since: 2021.4 + */ +typedef enum { + OSTREE_REPO_VERIFY_FLAGS_NONE = 0, + OSTREE_REPO_VERIFY_FLAGS_NO_GPG = (1 << 0), + OSTREE_REPO_VERIFY_FLAGS_NO_SIGNAPI = (1 << 1), +} OstreeRepoVerifyFlags; + +_OSTREE_PUBLIC +gboolean ostree_repo_signature_verify_commit_data (OstreeRepo *self, + const char *remote_name, + GBytes *commit_data, + GBytes *commit_metadata, + OstreeRepoVerifyFlags flags, + char **out_results, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_regenerate_summary (OstreeRepo *self, GVariant *additional_metadata, diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index c6c52382..8b2325d5 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -31,7 +31,10 @@ #include +static gboolean opt_verify; + static GOptionEntry options[] = { + { "verify", 'V', 0, G_OPTION_ARG_NONE, &opt_verify, "Print the commit verification status", NULL }, { NULL } }; @@ -86,6 +89,12 @@ deployment_print_status (OstreeSysroot *sysroot, g_autoptr(GVariant) commit_metadata = NULL; if (commit) commit_metadata = g_variant_get_child_value (commit, 0); + g_autoptr(GVariant) commit_detached_metadata = NULL; + if (commit) + { + if (!ostree_repo_read_commit_detached_metadata (repo, ref, &commit_detached_metadata, cancellable, error)) + return FALSE; + } const char *version = NULL; const char *source_title = NULL; @@ -139,7 +148,7 @@ deployment_print_status (OstreeSysroot *sysroot, } #ifndef OSTREE_DISABLE_GPGME - if (deployment_get_gpg_verify (deployment, repo)) + if (!opt_verify && deployment_get_gpg_verify (deployment, repo)) { g_autoptr(GString) output_buffer = g_string_sized_new (256); /* Print any digital signatures on this commit. */ @@ -172,6 +181,31 @@ deployment_print_status (OstreeSysroot *sysroot, g_print ("%s", output_buffer->str); } #endif /* OSTREE_DISABLE_GPGME */ + if (opt_verify) + { + if (!commit) + return glnx_throw (error, "Cannot verify, failed to load commit"); + + if (origin == NULL) + return glnx_throw (error, "Cannot verify deployment with no origin"); + + g_autofree char *refspec = g_key_file_get_string (origin, "origin", "refspec", NULL); + if (refspec == NULL) + return glnx_throw (error, "No origin/refspec, cannot verify"); + g_autofree char *remote = NULL; + if (!ostree_parse_refspec (refspec, &remote, NULL, NULL)) + return FALSE; + if (remote == NULL) + return glnx_throw (error, "Cannot verify deployment without remote"); + + g_autoptr(GBytes) commit_data = g_variant_get_data_as_bytes (commit); + g_autoptr(GBytes) commit_detached_metadata_bytes = + commit_detached_metadata ? g_variant_get_data_as_bytes (commit_detached_metadata) : NULL; + g_autofree char *verify_text = NULL; + if (!ostree_repo_signature_verify_commit_data (repo, remote, commit_data, commit_detached_metadata_bytes, 0, &verify_text, error)) + return FALSE; + g_print ("%s\n", verify_text); + } return TRUE; } diff --git a/tests/.gitignore b/tests/.gitignore index 938c169f..6355c8df 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -24,3 +24,4 @@ test-repo-finder-mount test-rfc2616-dates test-rollsum-cli test-kargs +test-commit-sign-sh-ext diff --git a/tests/test-admin-gpg.sh b/tests/test-admin-gpg.sh index 2167f673..bd34aae4 100755 --- a/tests/test-admin-gpg.sh +++ b/tests/test-admin-gpg.sh @@ -148,4 +148,9 @@ ${CMD_PREFIX} ostree admin status > status.txt test -f status.txt assert_file_has_content status.txt "GPG: Signature made" assert_not_file_has_content status.txt "GPG: Can't check signature: public key not found" +rm -f status.txt + +${CMD_PREFIX} ostree admin status --verify > status.txt +assert_file_has_content status.txt "GPG: Signature made" +rm -f status.txt echo 'ok gpg signature' diff --git a/tests/test-commit-sign-sh-ext.c b/tests/test-commit-sign-sh-ext.c new file mode 100644 index 00000000..b5c5dcc6 --- /dev/null +++ b/tests/test-commit-sign-sh-ext.c @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2021 Red Hat, Inc. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "libglnx.h" +#include + +static void +assert_error_contains (GError **error, const char *msg) +{ + g_assert (error != NULL); + GError *actual = *error; + g_assert (actual != NULL); + if (strstr (actual->message, msg) == NULL) + g_error ("%s does not contain %s", actual->message, msg); + g_clear_error (error); +} + +// Perhaps in the future we hook this up to a fuzzer +static GBytes * +corrupt (GBytes *input) +{ + gsize len = 0; + const guint8 *buf = g_bytes_get_data (input, &len); + g_assert_cmpint (len, >, 0); + g_assert_cmpint (len, <, G_MAXINT); + g_autofree char *newbuf = g_memdup (buf, len); + int o = g_random_int_range (0, len); + newbuf[o] = (newbuf[0] + 1); + + return g_bytes_new_take (g_steal_pointer (&newbuf), len); +} + +static gboolean +run (GError **error) +{ + g_autoptr(OstreeRepo) repo = ostree_repo_open_at (AT_FDCWD, "repo", NULL, error); + if (!repo) + return FALSE; + + g_autofree char *rev = NULL; + if (!ostree_repo_resolve_rev (repo, "origin:main", FALSE, &rev, error)) + return FALSE; + g_assert (rev); + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, &commit, error)) + return FALSE; + g_assert (commit); + + g_autoptr(GVariant) detached_meta = NULL; + if (!ostree_repo_read_commit_detached_metadata (repo, rev, &detached_meta, NULL, error)) + return FALSE; + g_assert (detached_meta); + + g_autoptr(GBytes) commit_bytes = g_variant_get_data_as_bytes (commit); + g_autoptr(GBytes) detached_meta_bytes = g_variant_get_data_as_bytes (detached_meta); + g_autofree char *verify_report = NULL; + if (!ostree_repo_signature_verify_commit_data (repo, "origin", commit_bytes, detached_meta_bytes, 0, + &verify_report, error)) + return FALSE; + + if (ostree_repo_signature_verify_commit_data (repo, "origin", commit_bytes, detached_meta_bytes, + OSTREE_REPO_VERIFY_FLAGS_NO_GPG | OSTREE_REPO_VERIFY_FLAGS_NO_SIGNAPI, + &verify_report, error)) + g_error ("Should not have validated"); + assert_error_contains (error, "No commit verification types enabled"); + + // No signatures + g_autoptr(GBytes) empty = g_bytes_new_static ("", 0); + if (ostree_repo_signature_verify_commit_data (repo, "origin", commit_bytes, empty, 0, + &verify_report, error)) + g_error ("Should not have validated"); + assert_error_contains (error, "no signatures found"); + // No such remote + if (ostree_repo_signature_verify_commit_data (repo, "nosuchremote", commit_bytes, detached_meta_bytes, 0, + &verify_report, error)) + g_error ("Should not have validated"); + assert_error_contains (error, "Remote \"nosuchremote\" not found"); + + // Corrupted commit + g_autoptr(GBytes) corrupted_commit = corrupt (commit_bytes); + if (ostree_repo_signature_verify_commit_data (repo, "origin", corrupted_commit, detached_meta_bytes, 0, + &verify_report, error)) + g_error ("Should not have validated"); + assert_error_contains (error, "BAD signature"); + + return TRUE; +} + +int +main (int argc, char **argv) +{ + g_autoptr(GError) error = NULL; + if (!run (&error)) + { + g_printerr ("error: %s\n", error->message); + exit (1); + } +} diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh index e9e7a6da..c3f9ce63 100755 --- a/tests/test-commit-sign.sh +++ b/tests/test-commit-sign.sh @@ -28,7 +28,7 @@ if ! has_gpgme; then exit 0 fi -echo "1..6" +echo "1..7" keyid="472CDAFA" oldpwd=`pwd` @@ -85,9 +85,15 @@ ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/g ${CMD_PREFIX} ostree --repo=repo pull origin main ${CMD_PREFIX} ostree --repo=repo show --gpg-verify-remote=origin main > show.txt assert_file_has_content_literal show.txt 'Found 1 signature' -rm repo -rf echo "ok pull verify" +# Run tests written in C +${OSTREE_UNINSTALLED}/tests/test-commit-sign-sh-ext +echo "ok extra C tests" + +# Clean things up and reinit +rm repo -rf + # A test with corrupted detached signature cd ${test_tmpdir} find ${test_tmpdir}/ostree-srv/gnomerepo -name '*.commitmeta' | while read fname; do From 27fcee861acb9fb1c517db684626ede30f560872 Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 30 Aug 2021 14:10:05 +0000 Subject: [PATCH 39/47] libtest: tweak selinux/relabel message --- tests/libtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index f04ccaa0..3976bc5b 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -609,7 +609,7 @@ have_systemd_and_libmount() { # https://github.com/ostreedev/ostree/pull/1217 skip_without_no_selinux_or_relabel () { if ! have_selinux_relabel; then - skip "this test requires xattr support" + skip "this test requires SELinux relabeling support" fi } From 9f5b636990d9a1fb0fc43cedcd96412c515fb679 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 26 Aug 2021 12:47:00 -0400 Subject: [PATCH 40/47] tests/basic: Skip --no-xattrs if we have selinux It cannot work to use `--no-xattrs` when SELinux is enabled because we get a `security.selinux` attribute on created files regardless. So just skip this test if true. Also add some `ostree fsck`s in here which helped me debug this. --- tests/basic-test.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 935544d9..b694f370 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -455,11 +455,18 @@ $OSTREE commit ${COMMIT_ARGS} --skip-if-unchanged -b trees/test2 -s 'should not $OSTREE ls -R -C test2 new_rev=$($OSTREE rev-parse test2) assert_streq "${old_rev}" "${new_rev}" +$OSTREE fsck echo "ok commit --skip-if-unchanged" cd ${test_tmpdir}/checkout-test2-4 +# Unfortunately later tests depend on this right now, so commit anyways $OSTREE commit ${COMMIT_ARGS} -b test2 -s "no xattrs" --no-xattrs -echo "ok commit with no xattrs" +if have_selinux_relabel; then + echo "ok # SKIP we get an injected security.selinux xattr regardless, so we can't do this" +else + $OSTREE fsck + echo "ok commit with no xattrs" +fi mkdir tree-A tree-B touch tree-A/file-a tree-B/file-b From aa0bb176c5847b3bc7096904c6b0e978d119c2fa Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 30 Aug 2021 14:11:43 +0000 Subject: [PATCH 41/47] tests/basic: avoid changing ownership This avoids possible issues when trying to chmod, tweaking permissions instead. --- tests/basic-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index b694f370..29ca6109 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -435,7 +435,7 @@ echo "ok user checkout" $OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit" --tree=ref=test2 echo "ok commit from ref" -$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit with modifier" --tree=ref=test2 --owner-uid=0 +$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Another commit with modifier" --tree=ref=test2 --mode-ro-executables echo "ok commit from ref with modifier" $OSTREE commit ${COMMIT_ARGS} -b trees/test2 -s 'ref with / in it' --tree=ref=test2 From fff24089ddc5893140b9a73fd8ea260df07172fe Mon Sep 17 00:00:00 2001 From: Luca BRUNO Date: Mon, 30 Aug 2021 15:55:16 +0000 Subject: [PATCH 42/47] tests: skip a broken fsck case There are some existing issues around fsck in unprivileged bare mode, so this test does not really work at the moment. Leaving it as a FIXME for the moment. --- tests/basic-test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 29ca6109..850a7605 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -793,7 +793,10 @@ rm files -rf && mkdir files mkdir files/worldwritable-dir chmod a+w files/worldwritable-dir $OSTREE commit ${COMMIT_ARGS} -b content-with-dir-world-writable --tree=dir=files -$OSTREE fsck +# FIXME(lucab): this seems to fail in unprivileged bare mode. +if ! have_selinux_relabel; then + $OSTREE fsck +fi rm dir-co -rf $OSTREE checkout -U -H -M content-with-dir-world-writable dir-co if is_bare_user_only_repo repo; then From 8821ec6e569c8e77712f964da6d886fcb665a0cc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 2 Sep 2021 16:59:15 -0400 Subject: [PATCH 43/47] upgrade: Stabilize deployment staging We're waaay overdue for this, it's been the default in rpm-ostree for years, and solves several important bugs around not capturing `/etc` while things are running. Also, `ostree admin upgrade --stage` (should) become idempotent. Closes: https://github.com/ostreedev/ostree/issues/2389 --- man/ostree-admin-upgrade.xml | 10 ++++++++++ src/libostree/ostree-sysroot-upgrader.c | 4 +++- src/libostree/ostree-sysroot-upgrader.h | 3 +++ src/ostree/ot-admin-builtin-upgrade.c | 10 ++++++++-- tests/kolainst/destructive/staged-deploy.sh | 5 +---- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/man/ostree-admin-upgrade.xml b/man/ostree-admin-upgrade.xml index 8d0cb438..002a2170 100644 --- a/man/ostree-admin-upgrade.xml +++ b/man/ostree-admin-upgrade.xml @@ -96,6 +96,16 @@ Boston, MA 02111-1307, USA. and is ready to deploy them. + + + + + Perform deployment finalization at shutdown time. Recommended, + and will likely become the default in the future. + + + + , diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 85e67340..eefeda6e 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -646,7 +646,8 @@ ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader *self, g_autoptr(OstreeDeployment) new_deployment = NULL; /* Experimental flag to enable staging */ - if (getenv ("OSTREE_EX_STAGE_DEPLOYMENTS")) + gboolean stage = (self->flags & OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE) > 0 || getenv ("OSTREE_EX_STAGE_DEPLOYMENTS") != NULL; + if (stage) { if (!ostree_sysroot_stage_tree (self->sysroot, self->osname, self->new_revision, @@ -688,6 +689,7 @@ ostree_sysroot_upgrader_flags_get_type (void) { static const GFlagsValue values[] = { { OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED, "OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED", "ignore-unconfigured" }, + { OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE, "OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE", "stage" }, { 0, NULL, NULL } }; GType g_define_type_id = diff --git a/src/libostree/ostree-sysroot-upgrader.h b/src/libostree/ostree-sysroot-upgrader.h index c9bf8a12..10a463c5 100644 --- a/src/libostree/ostree-sysroot-upgrader.h +++ b/src/libostree/ostree-sysroot-upgrader.h @@ -35,12 +35,15 @@ G_BEGIN_DECLS * OstreeSysrootUpgraderFlags: * @OSTREE_SYSROOT_UPGRADER_FLAGS_NONE: No options * @OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED: Do not error if the origin has an unconfigured-state key + * @OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE: Enable "staging" (finalization at shutdown); recommended + * (Since: 2021.4) * * Flags controlling operation of an #OstreeSysrootUpgrader. */ typedef enum { OSTREE_SYSROOT_UPGRADER_FLAGS_NONE = (1 << 0), OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED = (1 << 1), + OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE = (1 << 2), } OstreeSysrootUpgraderFlags; _OSTREE_PUBLIC diff --git a/src/ostree/ot-admin-builtin-upgrade.c b/src/ostree/ot-admin-builtin-upgrade.c index eafe8bce..2c0149c1 100644 --- a/src/ostree/ot-admin-builtin-upgrade.c +++ b/src/ostree/ot-admin-builtin-upgrade.c @@ -37,6 +37,7 @@ static gboolean opt_reboot; static gboolean opt_allow_downgrade; static gboolean opt_pull_only; static gboolean opt_deploy_only; +static gboolean opt_stage; static char *opt_osname; static char *opt_override_commit; @@ -47,6 +48,7 @@ static GOptionEntry options[] = { { "override-commit", 0, 0, G_OPTION_ARG_STRING, &opt_override_commit, "Deploy CHECKSUM instead of the latest tree", "CHECKSUM" }, { "pull-only", 0, 0, G_OPTION_ARG_NONE, &opt_pull_only, "Do not create a deployment, just download", NULL }, { "deploy-only", 0, 0, G_OPTION_ARG_NONE, &opt_deploy_only, "Do not pull, only deploy", NULL }, + { "stage", 0, 0, G_OPTION_ARG_NONE, &opt_stage, "Enable staging (finalization at reboot time)", NULL }, { NULL } }; @@ -74,9 +76,13 @@ ot_admin_builtin_upgrade (int argc, char **argv, OstreeCommandInvocation *invoca return FALSE; } + OstreeSysrootUpgraderFlags flags = 0; + if (opt_stage) + flags |= OSTREE_SYSROOT_UPGRADER_FLAGS_STAGE; + g_autoptr(OstreeSysrootUpgrader) upgrader = - ostree_sysroot_upgrader_new_for_os (sysroot, opt_osname, - cancellable, error); + ostree_sysroot_upgrader_new_for_os_with_flags (sysroot, opt_osname, flags, + cancellable, error); if (!upgrader) return FALSE; diff --git a/tests/kolainst/destructive/staged-deploy.sh b/tests/kolainst/destructive/staged-deploy.sh index b5d0b319..f55bb2c8 100755 --- a/tests/kolainst/destructive/staged-deploy.sh +++ b/tests/kolainst/destructive/staged-deploy.sh @@ -19,9 +19,6 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in commit=${host_commit} # Test the deploy --stage functionality; first, we stage a deployment # reboot, and validate that it worked. - # for now, until the preset propagates down - # Start up path unit - systemctl enable --now ostree-finalize-staged.path # Write staged-deploy commit cd /ostree/repo/tmp # https://github.com/ostreedev/ostree/issues/1569 @@ -70,7 +67,7 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in ostree checkout -H "${origcommit}" t ostree commit --no-bindings --parent="${origcommit}" -b staged-deploy -I --consume t newcommit=$(ostree rev-parse staged-deploy) - env OSTREE_EX_STAGE_DEPLOYMENTS=1 ostree admin upgrade >out.txt + ostree admin upgrade --stage >out.txt test -f /run/ostree/staged-deployment # Debating bouncing back out to Ansible for this firstdeploycommit=$(rpm-ostree status |grep 'Commit:' |head -1|sed -e 's,^ *Commit: *,,') From 744bf943691495aee91d4ce73db6a3ace95cc8c7 Mon Sep 17 00:00:00 2001 From: "Buddelmann, Richard RB" Date: Wed, 8 Sep 2021 08:41:05 +0200 Subject: [PATCH 44/47] repo-pull: legacy_transaction_resuming flag ignored for deltafiles the legacy_transaction_resuming flag is not used, which will mark the commit as done, even if files are missing. using already existing commitstate_is_partial function as fix --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 12409e63..388f782b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2439,7 +2439,7 @@ get_best_static_delta_start_for (OtPullData *pull_data, if (!ostree_repo_load_commit (pull_data->repo, to_revision, NULL, &to_rev_state, error)) return FALSE; - if (!(to_rev_state & OSTREE_REPO_COMMIT_STATE_PARTIAL)) + if (!(commitstate_is_partial(pull_data, to_rev_state))) { /* We already have this commit, we're done! */ out_result->result = DELTA_SEARCH_RESULT_UNCHANGED; From 511c7a13eeae0161f16a11033eec36252c6390ae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 7 Sep 2021 18:02:24 -0400 Subject: [PATCH 45/47] Add support for "custom remotes" This will be helpful for the "ostree native container" work in https://github.com/ostreedev/ostree-rs-ext/ Basically in order to reuse GPG/signapi verification, we need to support adding a remote, even though it can't be used via `ostree pull`. (At least, not until we merge ostree-rs-ext into ostree, but even then I think the principle stands) --- man/ostree.repo-config.xml | 8 ++++++++ src/libostree/ostree-repo-pull.c | 15 +++++++++++++++ src/libostree/ostree-repo.c | 30 ++++++++++++++++------------- src/ostree/ot-remote-builtin-add.c | 31 +++++++++++++++++++++++------- tests/pull-test.sh | 31 ++++++++++++++++++++++++++++-- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index 335f83e4..6e2bc7cc 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -355,6 +355,14 @@ Boston, MA 02111-1307, USA. If set, pulls from this remote will fail with the configured text. This is intended for OS vendors which have a subscription process to access content. + + custom-backend + If set, pulls from this remote via libostree will fail with an error that mentions the value. + It is recommended to make this a software identifier token (e.g. "examplecorp-fetcher"), not freeform text ("ExampleCorp Fetcher"). + This is intended to be used by higher level software that wants to fetch ostree commits via some other mechanism, while still reusing the core libostree infrastructure around e.g. signatures. + + + diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 12409e63..04a5359f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3965,6 +3965,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, else { g_autofree char *unconfigured_state = NULL; + g_autofree char *custom_backend = NULL; g_free (pull_data->remote_name); pull_data->remote_name = g_strdup (remote_name_or_baseurl); @@ -3996,6 +3997,20 @@ ostree_repo_pull_with_options (OstreeRepo *self, "remote unconfigured-state: %s", unconfigured_state); goto out; } + + if (!ostree_repo_get_remote_option (self, pull_data->remote_name, + "custom-backend", NULL, + &custom_backend, + error)) + goto out; + + if (custom_backend) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Cannot fetch via libostree - remote '%s' uses custom backend '%s'", + pull_data->remote_name, custom_backend); + goto out; + } } if (pull_data->remote_name && !(disable_sign_verify && disable_sign_verify_summary)) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0ff2c53f..42d2b0e0 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1589,7 +1589,6 @@ impl_repo_remote_add (OstreeRepo *self, GError **error) { g_return_val_if_fail (name != NULL, FALSE); - g_return_val_if_fail (url != NULL, FALSE); g_return_val_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}")), FALSE); if (!ostree_validate_remote_name (name, error)) @@ -1637,10 +1636,13 @@ impl_repo_remote_add (OstreeRepo *self, remote->file = g_file_get_child (etc_ostree_remotes_d, basename); } - if (g_str_has_prefix (url, "metalink=")) - g_key_file_set_string (remote->options, remote->group, "metalink", url + strlen ("metalink=")); - else - g_key_file_set_string (remote->options, remote->group, "url", url); + if (url) + { + if (g_str_has_prefix (url, "metalink=")) + g_key_file_set_string (remote->options, remote->group, "metalink", url + strlen ("metalink=")); + else + g_key_file_set_string (remote->options, remote->group, "url", url); + } if (options) keyfile_set_from_vardict (remote->options, remote->group, options); @@ -1676,7 +1678,7 @@ impl_repo_remote_add (OstreeRepo *self, * ostree_repo_remote_add: * @self: Repo * @name: Name of remote - * @url: URL for remote (if URL begins with metalink=, it will be used as such) + * @url: (allow-none): URL for remote (if URL begins with metalink=, it will be used as such) * @options: (allow-none): GVariant of type a{sv} * @cancellable: Cancellable * @error: Error @@ -1789,7 +1791,6 @@ impl_repo_remote_replace (OstreeRepo *self, GError **error) { g_return_val_if_fail (name != NULL, FALSE); - g_return_val_if_fail (url != NULL, FALSE); g_return_val_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}")), FALSE); if (!ostree_validate_remote_name (name, error)) @@ -1815,11 +1816,14 @@ impl_repo_remote_replace (OstreeRepo *self, if (!g_key_file_remove_group (remote->options, remote->group, error)) return FALSE; - if (g_str_has_prefix (url, "metalink=")) - g_key_file_set_string (remote->options, remote->group, "metalink", - url + strlen ("metalink=")); - else - g_key_file_set_string (remote->options, remote->group, "url", url); + if (url) + { + if (g_str_has_prefix (url, "metalink=")) + g_key_file_set_string (remote->options, remote->group, "metalink", + url + strlen ("metalink=")); + else + g_key_file_set_string (remote->options, remote->group, "url", url); + } if (options != NULL) keyfile_set_from_vardict (remote->options, remote->group, options); @@ -1866,7 +1870,7 @@ impl_repo_remote_replace (OstreeRepo *self, * @sysroot: (allow-none): System root * @changeop: Operation to perform * @name: Name of remote - * @url: URL for remote (if URL begins with metalink=, it will be used as such) + * @url: (allow-none): URL for remote (if URL begins with metalink=, it will be used as such) * @options: (allow-none): GVariant of type a{sv} * @cancellable: Cancellable * @error: Error diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 61539ec1..b08153ec 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -35,6 +35,7 @@ static char *opt_gpg_import; static char **opt_sign_verify; static char *opt_contenturl; static char *opt_collection_id; +static char *opt_custom_backend; static char *opt_sysroot; static char *opt_repo; @@ -51,6 +52,7 @@ static GOptionEntry option_entries[] = { { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "force", 0, 0, G_OPTION_ARG_NONE, &opt_force, "Replace the provided remote if it exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, + { "custom-backend", 0, 0, G_OPTION_ARG_STRING, &opt_custom_backend, "This remote has content not fetched via libostree", "NAME" }, { "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" }, { "collection-id", 0, 0, G_OPTION_ARG_STRING, &opt_collection_id, "Globally unique ID for this repository as an collection of refs for redistribution to other repositories", "COLLECTION-ID" }, @@ -103,7 +105,7 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio g_autoptr(OstreeRepo) repo = NULL; g_autoptr(GString) sign_verify = NULL; const char *remote_name; - const char *remote_url; + const char *remote_url = NULL; g_autoptr(GVariantBuilder) optbuilder = NULL; g_autoptr(GVariant) options = NULL; gboolean ret = FALSE; @@ -119,11 +121,26 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio cancellable, error)) goto out; - if (argc < 3) + if (opt_custom_backend) { - ot_util_usage_error (context, "NAME and URL must be specified", error); - goto out; + if (argc < 2) + { + ot_util_usage_error (context, "NAME must be specified", error); + goto out; + } + if (argc >= 3) + remote_url = argv[2]; } + else + { + if (argc < 3) + { + ot_util_usage_error (context, "NAME and URL must be specified", error); + goto out; + } + remote_url = argv[2]; + } + remote_name = argv[1]; if (opt_if_not_exists && opt_force) { @@ -133,9 +150,6 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio goto out; } - remote_name = argv[1]; - remote_url = argv[2]; - optbuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); if (argc > 3) @@ -159,6 +173,9 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio if (opt_contenturl != NULL) g_variant_builder_add (optbuilder, "{s@v}", "contenturl", g_variant_new_variant (g_variant_new_string (opt_contenturl))); + if (opt_custom_backend != NULL) + g_variant_builder_add (optbuilder, "{s@v}", + "custom-backend", g_variant_new_variant (g_variant_new_string (opt_custom_backend))); for (char **iter = opt_set; iter && *iter; iter++) { diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 274bd978..fcc22812 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -55,10 +55,10 @@ function verify_initial_contents() { } if has_gpgme; then - echo "1..37" + echo "1..38" else # 3 tests needs GPG support - echo "1..34" + echo "1..35" fi # Try both syntaxes @@ -598,6 +598,33 @@ assert_file_has_content err.txt "ONE BILLION DOLLARS" echo "ok unconfigured" +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=repo remote add --custom-backend=ostree-rs-ext fromcontainer +if ${CMD_PREFIX} ostree --repo=repo pull fromcontainer 2>err.txt; then + assert_not_reached "pull unexpectedly succeeded?" +fi +assert_file_has_content err.txt "remote 'fromcontainer' uses custom backend 'ostree-rs-ext'" + +for x in show-url refs; do +if ${CMD_PREFIX} ostree --repo=repo remote "$x" fromcontainer 2>err.txt; then +assert_file_has_content err.txt "remote 'fromcontainer' uses custom backend 'ostree-rs-ext'" + assert_not_reached "no url expected" +fi +assert_file_has_content err.txt "No \"url\" option in remote" +done +${CMD_PREFIX} ostree --repo=repo remote delete fromcontainer + +${CMD_PREFIX} ostree --repo=repo remote add --custom-backend=ostree-rs-ext fromcontainer2 docker://quay.io/examplecorp/foo +if ${CMD_PREFIX} ostree --repo=repo pull fromcontainer2 main 2>err.txt; then + assert_not_reached "pull unexpectedly succeeded?" +fi +assert_file_has_content err.txt "remote 'fromcontainer2' uses custom backend 'ostree-rs-ext'" +${CMD_PREFIX} ostree --repo=repo remote show-url fromcontainer2 >out.txt +assert_file_has_content out.txt docker://quay.io/examplecorp/foo +${CMD_PREFIX} ostree --repo=repo remote delete fromcontainer2 + +echo "ok custom backend" + cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo remote add origin-bad $(cat httpd-address)/ostree/noent From 55090f108d50a83c81b1e22c883dbfe3000dbace Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 8 Sep 2021 14:00:12 -0400 Subject: [PATCH 46/47] Release 2021.4 --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 6 ------ src/libostree/libostree-released.sym | 7 +++++++ tests/test-symbols.sh | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 4002a8c7..1c528b19 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ m4_define([year_version], [2021]) m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 7e6f7784..e3cd14a4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -22,12 +22,6 @@ - uncomment the include in Makefile-libostree.am */ -LIBOSTREE_2021.4 { -global: - ostree_repo_remote_get_gpg_keys; - ostree_repo_signature_verify_commit_data; -} LIBOSTREE_2021.3; - /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index becdfc55..1e359bfb 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -661,6 +661,13 @@ global: ostree_repo_lock_pop; } LIBOSTREE_2021.2; + +LIBOSTREE_2021.4 { +global: + ostree_repo_remote_get_gpg_keys; + ostree_repo_signature_verify_commit_data; +} LIBOSTREE_2021.3; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 89c3eb07..0de87be4 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -56,7 +56,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt < Date: Wed, 8 Sep 2021 14:01:17 -0400 Subject: [PATCH 47/47] configure: post-release version bump --- Makefile-libostree.am | 6 +++--- configure.ac | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index d40de48d..dd396974 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -173,9 +173,9 @@ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym # Uncomment this include when adding new development symbols. -if BUILDOPT_IS_DEVEL_BUILD -symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -endif +#if BUILDOPT_IS_DEVEL_BUILD +#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +#endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= diff --git a/configure.ac b/configure.ac index 1c528b19..5678145a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.63]) dnl To perform a release, follow the instructions in `docs/CONTRIBUTING.md`. m4_define([year_version], [2021]) -m4_define([release_version], [4]) +m4_define([release_version], [5]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux])