From a8fd37b6a08540d93c99d099a32ff5c3f89a99ec Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 17 May 2017 11:41:54 -0400 Subject: [PATCH 01/86] pull: complete detached meta fetch before scanning If somehow a repo has gpg verification on but doesn't have signatures present for the existing commit, ostree would error out if it needs to scan the commit object (e.g. if there are no updates available). An instance of this is currently happening in Fedora AH, in which signatures are not shipped in the ISO due to filesystem restrictions. Another possible scenario is if a content provider switches from not signing commits to signing them; even if older commits are retroactively signed, clients' local commit objects would error out if they needed scanning. This patch adds a check to ensure that we always attempt to fetch the detached metadata and wait for its result (whether it exists or not) before moving on to scan their corresponding commit objects. See also: https://github.com/projectatomic/rpm-ostree/issues/630 Closes: #873 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 38 +++++++++++++++++++------- tests/pull-test.sh | 46 ++++++++++++++++++++++++-------- tests/test-remote-headers.sh | 4 +-- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f1dae995..08692c59 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -80,6 +80,7 @@ typedef struct { GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */ GHashTable *commit_to_depth; /* Maps commit checksum maximum depth */ GHashTable *scanned_metadata; /* Maps object name to itself */ + GHashTable *fetched_detached_metadata; /* Set */ GHashTable *requested_metadata; /* Maps object name to itself */ GHashTable *requested_content; /* Maps checksum to itself */ GHashTable *requested_fallback_content; /* Maps checksum to itself */ @@ -912,8 +913,15 @@ meta_fetch_on_complete (GObject *object, { /* There isn't any detached metadata, just fetch the commit */ g_clear_error (&local_error); + + /* Now that we've at least tried to fetch it, we can proceed to + * scan/fetch the commit object */ + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (checksum)); + if (!fetch_data->object_is_stored) enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE); + else + queue_scan_one_metadata_object (pull_data, checksum, objtype, fetch_data->path, 0); } /* When traversing parents, do not fail on a missing commit. @@ -960,8 +968,12 @@ meta_fetch_on_complete (GObject *object, pull_data->cancellable, error)) goto out; + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (checksum)); + if (!fetch_data->object_is_stored) enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE); + else + queue_scan_one_metadata_object (pull_data, checksum, objtype, fetch_data->path, 0); } else { @@ -977,7 +989,7 @@ meta_fetch_on_complete (GObject *object, if (!write_commitpartial_for (pull_data, checksum, error)) goto out; } - + ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata, pull_data->cancellable, on_metadata_written, fetch_data); @@ -1377,15 +1389,20 @@ scan_one_metadata_object_c (OtPullData *pull_data, } else if (is_stored && objtype == OSTREE_OBJECT_TYPE_COMMIT) { - /* For commits, always refetch detached metadata. */ - enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, TRUE, TRUE); + /* Even though we already have the commit, we always try to (re)fetch the + * detached metadata before scanning it, in case new signatures appear. + * https://github.com/projectatomic/rpm-ostree/issues/630 */ + if (!g_hash_table_contains (pull_data->fetched_detached_metadata, tmp_checksum)) + enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, TRUE, TRUE); + else + { + if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, + pull_data->cancellable, error)) + goto out; - if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, - pull_data->cancellable, error)) - goto out; - - g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object)); - pull_data->n_scanned_metadata++; + g_hash_table_add (pull_data->scanned_metadata, g_variant_ref (object)); + pull_data->n_scanned_metadata++; + } } else if (is_stored && objtype == OSTREE_OBJECT_TYPE_DIR_TREE) { @@ -2787,6 +2804,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_free); pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); + pull_data->fetched_detached_metadata = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify)g_free, NULL); pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); pull_data->requested_fallback_content = g_hash_table_new_full (g_str_hash, g_str_equal, @@ -3509,6 +3528,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->commit_to_depth, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->fetched_detached_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref); diff --git a/tests/pull-test.sh b/tests/pull-test.sh index f81d8023..3a836da9 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -24,7 +24,7 @@ function repo_init() { rm repo -rf mkdir repo ostree_repo_init repo - ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@" } function verify_initial_contents() { @@ -35,10 +35,10 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..16" +echo "1..18" # Try both syntaxes -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main ${CMD_PREFIX} ostree --repo=repo pull origin:main ${CMD_PREFIX} ostree --repo=repo fsck @@ -128,7 +128,7 @@ assert_file_has_content main.txt ${rev} echo "ok pull specific commit" cd ${test_tmpdir} -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main ${CMD_PREFIX} ostree --repo=repo fsck # Generate a delta from old to current, even though we aren't going to @@ -153,7 +153,7 @@ ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u # Explicitly test delta fetches via ref name as well as commit hash for delta_target in main ${new_rev}; do cd ${test_tmpdir} -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${delta_target} >dry-run-pull.txt # Compression can vary, so we support 400-699 @@ -166,7 +166,7 @@ done # Explicitly test delta fetches via ref name as well as commit hash for delta_target in main ${new_rev}; do cd ${test_tmpdir} -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${delta_target} if test ${delta_target} = main; then @@ -179,7 +179,7 @@ ${CMD_PREFIX} ostree --repo=repo fsck done cd ${test_tmpdir} -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin main ${CMD_PREFIX} ostree --repo=repo fsck @@ -197,7 +197,7 @@ cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt ${CMD_PREFIX} ostree --repo=repo fsck @@ -211,7 +211,7 @@ echo "ok pull byteswapped delta" cd ${test_tmpdir} rm ostree-srv/gnomerepo/deltas -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u -repo_init +repo_init --no-gpg-verify if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" fi @@ -219,7 +219,7 @@ assert_file_has_content err.txt "deltas required, but none found" ${CMD_PREFIX} ostree --repo=repo fsck # Now test with a partial commit -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main@${prev_rev} if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" @@ -227,7 +227,7 @@ fi assert_file_has_content err.txt "deltas required, but none found" echo "ok delta required but don't exist" -repo_init +repo_init --no-gpg-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${new_rev} 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" @@ -294,3 +294,27 @@ fi assert_file_has_content err.txt "ONE BILLION DOLLARS" echo "ok unconfigured" + +cd ${test_tmpdir} +repo_init --set=gpg-verify=true +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit \ + --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} -b main \ + -s "A signed commit" --tree=ref=main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +# make sure gpg verification is correctly on +csum=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo rev-parse main) +objpath=objects/${csum::2}/${csum:2}.commitmeta +remotesig=ostree-srv/gnomerepo/$objpath +localsig=repo/$objpath +mv $remotesig $remotesig.bak +if ${CMD_PREFIX} ostree --repo=repo --depth=0 pull origin main; then + assert_not_reached "pull with gpg-verify unexpectedly succeeded?" +fi +# ok now check that we can pull correctly +mv $remotesig.bak $remotesig +${CMD_PREFIX} ostree --repo=repo pull origin main +echo "ok pull signed commit" +rm $localsig +${CMD_PREFIX} ostree --repo=repo pull origin main +test -f $localsig +echo "ok re-pull signature for stored commit" diff --git a/tests/test-remote-headers.sh b/tests/test-remote-headers.sh index 6902cefb..39fbe352 100755 --- a/tests/test-remote-headers.sh +++ b/tests/test-remote-headers.sh @@ -44,9 +44,9 @@ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat # Sanity check the setup, without headers the pull should fail assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main -echo "ok, setup done" +echo "ok setup done" # Now pull should succeed now ${CMD_PREFIX} ostree --repo=repo pull --http-header foo=bar --http-header baz=badger origin main -echo "ok, pull succeeded" +echo "ok pull succeeded" From d2eaded90de24fa17649d4812e92005f75694d22 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 18 May 2017 08:11:32 +0100 Subject: [PATCH 02/86] lib/remote: Add a getter for OstreeRemote.name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we’ve got a public, sealed OstreeRemote structure, we can start carefully exposing members of it as API. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- apidoc/ostree-experimental-sections.txt | 1 + src/libostree/libostree-experimental.sym | 5 +++++ src/libostree/ostree-remote.c | 20 ++++++++++++++++++++ src/libostree/ostree-remote.h | 3 +++ 4 files changed, 29 insertions(+) diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 790feb33..998a4f9c 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -3,4 +3,5 @@ OstreeRemote ostree_remote_ref ostree_remote_unref +ostree_remote_get_name diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index e2ab4ea8..1d4feeda 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -29,3 +29,8 @@ global: ostree_remote_ref; ostree_remote_unref; } LIBOSTREE_2017.6; + +LIBOSTREE_2016.7_EXPERIMENTAL { +global: + ostree_remote_get_name; +} LIBOSTREE_2017.6_EXPERIMENTAL; diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c index 86fae8d8..f34fafa4 100644 --- a/src/libostree/ostree-remote.c +++ b/src/libostree/ostree-remote.c @@ -148,3 +148,23 @@ G_DEFINE_BOXED_TYPE(OstreeRemote, ostree_remote, ostree_remote_ref, ostree_remote_unref); #endif + +/** + * ostree_remote_get_name: + * @remote: an #OstreeRemote + * + * Get the human-readable name of the remote. This is what the user configured, + * if the remote was explicitly configured; and will otherwise be a stable, + * arbitrary, string. + * + * Returns: remote’s name + * Since: 2017.7 + */ +const gchar * +ostree_remote_get_name (OstreeRemote *remote) +{ + g_return_val_if_fail (remote != NULL, NULL); + g_return_val_if_fail (remote->ref_count > 0, NULL); + + return remote->name; +} diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h index 8e96213c..5505bc1d 100644 --- a/src/libostree/ostree-remote.h +++ b/src/libostree/ostree-remote.h @@ -60,4 +60,7 @@ void ostree_remote_unref (OstreeRemote *remote); #endif /* GI_SCANNER */ #endif +_OSTREE_PUBLIC +const gchar *ostree_remote_get_name (OstreeRemote *remote); + G_END_DECLS From 2910b880812362fcc57cf30c4b30f0c450389352 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 18 May 2017 08:16:18 +0100 Subject: [PATCH 03/86] lib/remote: Add internal annotations to OstreeRemote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just for internal documentation; g-ir-scanner doesn’t read or understand them. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- src/libostree/ostree-remote-private.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-remote-private.h b/src/libostree/ostree-remote-private.h index e207ed4c..13f757ed 100644 --- a/src/libostree/ostree-remote-private.h +++ b/src/libostree/ostree-remote-private.h @@ -38,9 +38,9 @@ G_BEGIN_DECLS struct OstreeRemote { volatile int ref_count; - char *name; - char *group; /* group name in options */ - char *keyring; /* keyring name (NAME.trustedkeys.gpg) */ + char *name; /* (not nullable) */ + char *group; /* group name in options (not nullable) */ + char *keyring; /* keyring name (NAME.trustedkeys.gpg) (not nullable) */ GFile *file; /* NULL if remote defined in repo/config */ GKeyFile *options; }; From ed7905d000f6d20374e3cde4d85281fb5bfdcdf9 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 18 May 2017 08:17:29 +0100 Subject: [PATCH 04/86] lib/remote: Add arguments to internal OstreeRemote constructor Add a name argument to the internal OstreeRemote constructor, since this member (and several derived from it) is non-nullable, and hence must always be set at construction time. This changes the only call sites of the constructor to use the new API, which is internal. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- src/libostree/ostree-remote-private.h | 2 +- src/libostree/ostree-remote.c | 14 +++++++++----- src/libostree/ostree-repo.c | 5 +---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libostree/ostree-remote-private.h b/src/libostree/ostree-remote-private.h index 13f757ed..856cdaa6 100644 --- a/src/libostree/ostree-remote-private.h +++ b/src/libostree/ostree-remote-private.h @@ -46,7 +46,7 @@ struct OstreeRemote { }; G_GNUC_INTERNAL -OstreeRemote *ostree_remote_new (void); +OstreeRemote *ostree_remote_new (const gchar *name); G_GNUC_INTERNAL OstreeRemote *ostree_remote_new_from_keyfile (GKeyFile *keyfile, diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c index f34fafa4..1932b22f 100644 --- a/src/libostree/ostree-remote.c +++ b/src/libostree/ostree-remote.c @@ -53,12 +53,17 @@ */ OstreeRemote * -ostree_remote_new (void) +ostree_remote_new (const gchar *name) { OstreeRemote *remote; + g_return_val_if_fail (name != NULL && *name != '\0', NULL); + remote = g_slice_new0 (OstreeRemote); remote->ref_count = 1; + remote->name = g_strdup (name); + remote->group = g_strdup_printf ("remote \"%s\"", name); + remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", name); remote->options = g_key_file_new (); return remote; @@ -70,6 +75,7 @@ ostree_remote_new_from_keyfile (GKeyFile *keyfile, { g_autoptr(GMatchInfo) match = NULL; OstreeRemote *remote; + g_autofree gchar *name = NULL; static gsize regex_initialized; static GRegex *regex; @@ -88,10 +94,8 @@ ostree_remote_new_from_keyfile (GKeyFile *keyfile, if (!g_regex_match (regex, group, 0, &match)) return NULL; - remote = ostree_remote_new (); - remote->name = g_match_info_fetch (match, 1); - remote->group = g_strdup (group); - remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", remote->name); + name = g_match_info_fetch (match, 1); + remote = ostree_remote_new (name); ot_keyfile_copy_group (keyfile, remote->options, group); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2ef64ec0..b47bd382 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -882,10 +882,7 @@ impl_repo_remote_add (OstreeRepo *self, name, remote->file ? gs_file_get_path_cached (remote->file) : "(in config)"); } - remote = ostree_remote_new (); - remote->name = g_strdup (name); - remote->group = g_strdup_printf ("remote \"%s\"", name); - remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", name); + remote = ostree_remote_new (name); /* The OstreeRepo maintains its own internal system root path, * so we need to not only check if a "sysroot" argument was given From b6ac28b0dac0f4dafb7aa671e53bd57db72cc2ed Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 18 May 2017 08:26:34 +0100 Subject: [PATCH 05/86] lib/repo: Add return value to _ostree_repo_add_remote() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return whether the remote already existed. This is an internal API, so it’s not an API break. The return value will be useful in upcoming commits for working out whether to later remove a remote again. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 2 +- src/libostree/ostree-repo.c | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 2a518d4f..d3955ad5 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -351,7 +351,7 @@ gboolean _ostree_repo_update_mtime (OstreeRepo *self, GError **error); -void +gboolean _ostree_repo_add_remote (OstreeRepo *self, OstreeRemote *remote); OstreeRemote * diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b47bd382..a5cfb123 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -150,19 +150,23 @@ _ostree_repo_get_remote_inherited (OstreeRepo *self, return g_steal_pointer (&remote); } -void +gboolean _ostree_repo_add_remote (OstreeRepo *self, OstreeRemote *remote) { - g_return_if_fail (self != NULL); - g_return_if_fail (remote != NULL); - g_return_if_fail (remote->name != NULL); + gboolean already_existed; + + g_return_val_if_fail (self != NULL, FALSE); + g_return_val_if_fail (remote != NULL, FALSE); + g_return_val_if_fail (remote->name != NULL, FALSE); g_mutex_lock (&self->remotes_lock); - g_hash_table_replace (self->remotes, remote->name, ostree_remote_ref (remote)); + already_existed = g_hash_table_replace (self->remotes, remote->name, ostree_remote_ref (remote)); g_mutex_unlock (&self->remotes_lock); + + return already_existed; } static gboolean From 242a0fd77928bb3fa1c960a78936b3d43a99c4c6 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 18 May 2017 08:28:03 +0100 Subject: [PATCH 06/86] lib/repo: Make ost_repo_remove_remote() available internally Make it an internal, not static, API; like _ostree_repo_add_remote(). It will be used in many the same situations. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 3 +++ src/libostree/ostree-repo.c | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index d3955ad5..7efbf91f 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -354,6 +354,9 @@ _ostree_repo_update_mtime (OstreeRepo *self, gboolean _ostree_repo_add_remote (OstreeRepo *self, OstreeRemote *remote); +gboolean +_ostree_repo_remove_remote (OstreeRepo *self, + OstreeRemote *remote); OstreeRemote * _ostree_repo_get_remote (OstreeRepo *self, const char *name, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index a5cfb123..7ee2c9aa 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -169,9 +169,9 @@ _ostree_repo_add_remote (OstreeRepo *self, return already_existed; } -static gboolean -ost_repo_remove_remote (OstreeRepo *self, - OstreeRemote *remote) +gboolean +_ostree_repo_remove_remote (OstreeRepo *self, + OstreeRemote *remote) { gboolean removed; @@ -1040,7 +1040,7 @@ impl_repo_remote_delete (OstreeRepo *self, if (!ot_ensure_unlinked_at (self->repo_dir_fd, remote->keyring, error)) return FALSE; - ost_repo_remove_remote (self, remote); + _ostree_repo_remove_remote (self, remote); return TRUE; } From 1feda846efe1b4848ebfb00a2a3a9d625b4d7edc Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 19 May 2017 10:11:35 +0100 Subject: [PATCH 07/86] lib/remote: Fix compilation with --enable-experimental-api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole ostree-remote.h file is only included in the public ostree.h header if OSTREE_ENABLE_EXPERIMENTAL_API is defined, so there’s no need to change the set of methods defined in it according to whether we’re compiling with experimental API. Signed-off-by: Philip Withnall Closes: #875 Approved by: cgwalters --- src/libostree/ostree-remote.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h index 5505bc1d..96fd4556 100644 --- a/src/libostree/ostree-remote.h +++ b/src/libostree/ostree-remote.h @@ -48,17 +48,14 @@ G_BEGIN_DECLS typedef struct OstreeRemote OstreeRemote; #endif -#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#ifndef __GI_SCANNER__ _OSTREE_PUBLIC GType ostree_remote_get_type (void) G_GNUC_CONST; -#else -#ifndef __GI_SCANNER__ _OSTREE_PUBLIC OstreeRemote *ostree_remote_ref (OstreeRemote *remote); _OSTREE_PUBLIC void ostree_remote_unref (OstreeRemote *remote); #endif /* GI_SCANNER */ -#endif _OSTREE_PUBLIC const gchar *ostree_remote_get_name (OstreeRemote *remote); From 84d6267b613b2c71dd21563e62c39797aa498699 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 May 2017 11:00:34 -0400 Subject: [PATCH 08/86] tests/test-symbols.sh: Fix with --enable-experimental-api We missed a `--no-filename` for grep with the documentation. Closes: #875 Approved by: cgwalters --- tests/test-symbols.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 3ee018b0..54f469fb 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -39,7 +39,7 @@ echo "ok exports" grep -E -v '(ostree_cmd__private__)|(ostree_fetcher_config_flags_get_type)' found-symbols.txt > expected-documented.txt echo "Verifying all public symbols are documented:" -grep '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt $experimental_sections |sort -u > found-documented.txt +grep --no-filename '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt $experimental_sections |sort -u > found-documented.txt diff -u expected-documented.txt found-documented.txt echo 'ok documented symbols' From add88c3a23d3bddd7f42d6bbd5601686c7c47e1e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 May 2017 10:39:48 -0400 Subject: [PATCH 09/86] ci: Add unit case for --enable-experimental-api We're starting to get a lot of contexts, and this is likely going to drive some requirements for consolidation and improvements like not testing *every* context on every PR, etc. But for now since experimental is new, and under development, let's test it. Closes: #875 Approved by: cgwalters --- .redhat-ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 97bbcf87..cc02f75e 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -76,6 +76,30 @@ env: tests: - make check TESTS=tests/test-rollsum +artifacts: + - test-suite.log +--- + +inherit: true + +context: f25-experimental-api + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-gtk-doc + --enable-experimental-api + +env: + CC: 'gcc' + +tests: + - make check + - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' + +artifacts: + - test-suite.log --- inherit: true From a2be46114abeda76619d5a428c3106ac6b7b3f61 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 May 2017 13:10:45 -0400 Subject: [PATCH 10/86] tests/libtest-core: Copy rpm-ostree changes, clean up I want to keep this a "pure copy-able" file into various projects like rpm-ostree, bwrap, and flatpak. Pull in changes from rpm-ostree to prep for that. While we have the patient open, dedup the code for file matching a bit. Closes: #877 Approved by: jlebon --- tests/libtest-core.sh | 54 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index d1d3bbdf..14d9cd5c 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -1,4 +1,8 @@ -# Core source library for shell script tests +# Core source library for shell script tests; this +# file is intended to be the canonical source, which at +# is copied at least into: +# +# - https://github.com/projectatomic/rpm-ostree # # Copyright (C) 2017 Colin Walters # @@ -36,6 +40,7 @@ fi # This should really be the default IMO export G_DEBUG=fatal-warnings + assert_streq () { test "$1" = "$2" || fatal "$1 != $2" } @@ -58,18 +63,29 @@ assert_has_dir () { test -d "$1" || fatal "Couldn't find '$1'" } +# Dump ls -al + file contents to stderr, then fatal() +_fatal_print_file() { + file="$1" + shift + ls -al "$file" >&2 + sed -e 's/^/# /' < "$file" >&2 + fatal "$@" +} + assert_not_has_file () { if test -f "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' exists" + _fatal_print_file "$1" "File '$1' exists" fi } assert_not_file_has_content () { - if grep -q -e "$2" "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' incorrectly matches regexp '$2'" - fi + fpath=$1 + shift + for re in "$@"; do + if grep -q -e "$re" "$fpath"; then + _fatal_print_file "$fpath" "File '$fpath' matches regexp '$re'" + fi + done } assert_not_has_dir () { @@ -79,35 +95,33 @@ assert_not_has_dir () { } assert_file_has_content () { - if ! grep -q -e "$2" "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' doesn't match regexp '$2'" - fi + fpath=$1 + shift + for re in "$@"; do + if ! grep -q -e "$re" "$fpath"; then + _fatal_print_file "$fpath" "File '$fpath' doesn't match regexp '$re'" + fi + done } assert_file_has_content_literal () { if ! grep -q -F -e "$2" "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' doesn't match fixed string list '$2'" + _fatal_print_file "$1" "File '$1' doesn't match fixed string list '$2'" fi } assert_symlink_has_content () { if ! test -L "$1"; then - echo 1>&2 "File '$1' is not a symbolic link" - exit 1 + fatal "File '$1' is not a symbolic link" fi if ! readlink "$1" | grep -q -e "$2"; then - sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "Symbolic link '$1' doesn't match regexp '$2'" - exit 1 + _fatal_print_file "$1" "Symbolic link '$1' doesn't match regexp '$2'" fi } assert_file_empty() { if test -s "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' is not empty" + _fatal_print_file "$1" "File '$1' is not empty" fi } From cd65f85dcbed6930eb5d3fdb344b173c4356da30 Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Wed, 3 May 2017 09:36:48 -0600 Subject: [PATCH 11/86] libtest: allow committing to alternative branches This will be used by the upcoming test-admin-upgrade-endoflife.sh Closes: #874 Approved by: cgwalters --- tests/libtest.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 3ce718f9..1a81c755 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -446,6 +446,7 @@ os_repository_new_commit () { boot_checksum_iteration=${1:-0} content_iteration=${2:-0} + branch=${3:-testos/buildmaster/x86_64-runtime} echo "BOOT ITERATION: $boot_checksum_iteration" cd ${test_tmpdir}/osdata rm boot/* @@ -464,7 +465,7 @@ os_repository_new_commit () version=$(date "+%Y%m%d.${content_iteration}") - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" -b testos/buildmaster/x86_64-runtime -s "Build" + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" -b $branch -s "Build" cd ${test_tmpdir} } From c1ed9a15c1adb44ef30984f984c7ce199b88af03 Mon Sep 17 00:00:00 2001 From: Daniel Drake Date: Mon, 15 May 2017 12:13:38 -0600 Subject: [PATCH 12/86] Allow commits to mark refs as EOL, replaced by others A commit can now include a "ostree.endoflife-rebase" metadata key pointing to a new ref. When updating, the sysroot upgrader will see this and proceed to pull and deploy the new ref instead. The origin file in the new deployment will point to the new ref. This functionality is planned to be used in Endless OS. We will create a lesser tested branch for brand new, cutting edge hardware support, and ship that on hardware platforms that require the latest drivers. However, once our slower-moving official release is later updated to support the new hardware, we will use this functionality to migrate those bleeding-edge users over to the official release. Closes: #874 Approved by: cgwalters --- Makefile-tests.am | 1 + src/libostree/ostree-sysroot-upgrader.c | 46 ++++++++++++++++ tests/test-admin-upgrade-endoflife.sh | 72 +++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100755 tests/test-admin-upgrade-endoflife.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 89675288..6aae5f05 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -82,6 +82,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ tests/test-admin-upgrade-unconfigured.sh \ + tests/test-admin-upgrade-endoflife.sh \ tests/test-admin-deploy-syslinux.sh \ tests/test-admin-deploy-2.sh \ tests/test-admin-deploy-karg.sh \ diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 9816b3d6..45ef90e6 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -512,6 +512,10 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, char *refs_to_fetch[] = { NULL, NULL }; const char *from_revision = NULL; g_autofree char *origin_refspec = NULL; + g_autofree char *new_revision = NULL; + g_autoptr(GVariant) new_variant = NULL; + g_autoptr(GVariant) new_metadata = NULL; + g_autoptr(GVariant) rebase = NULL; if (self->override_csum != NULL) refs_to_fetch[0] = self->override_csum; @@ -541,6 +545,48 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, ostree_async_progress_finish (progress); } + /* Check to see if the commit marks the ref as EOL, redirecting to + * another. */ + if (!ostree_repo_resolve_rev (repo, origin_refspec, FALSE, + &new_revision, error)) + return FALSE; + + if (!ostree_repo_load_variant (repo, + OSTREE_OBJECT_TYPE_COMMIT, + new_revision, + &new_variant, + error)) + return FALSE; + + g_variant_get_child (new_variant, 0, "@a{sv}", &new_metadata); + rebase = g_variant_lookup_value (new_metadata, "ostree.endoflife-rebase", G_VARIANT_TYPE_STRING); + if (rebase) + { + const char *new_ref = g_variant_get_string (rebase, 0); + + /* Pull the new ref */ + if (self->origin_remote && + (upgrader_flags & OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC) == 0) + { + refs_to_fetch[0] = (char *) new_ref; + if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch, + flags, progress, cancellable, error)) + return FALSE; + } + + /* Use the new ref for the rest of the update process */ + g_free (self->origin_ref); + self->origin_ref = g_strdup(new_ref); + g_free (origin_refspec); + + if (self->origin_remote) + origin_refspec = g_strconcat (self->origin_remote, ":", new_ref, NULL); + else + origin_refspec = g_strdup (new_ref); + + g_key_file_set_string (self->origin, "origin", "refspec", origin_refspec); + } + if (self->override_csum != NULL) { if (!ostree_repo_set_ref_immediate (repo, diff --git a/tests/test-admin-upgrade-endoflife.sh b/tests/test-admin-upgrade-endoflife.sh new file mode 100755 index 00000000..a58c64ce --- /dev/null +++ b/tests/test-admin-upgrade-endoflife.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright (C) 2014 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +# Exports OSTREE_SYSROOT so --sysroot not needed. +setup_os_repository "archive-z2" "syslinux" +# This does: +# - init ostree repo in testos-repo +# - create system files in osdata and commit twice those contents into testos-repo +# - copy osdata to osdata-devel and make another change then commit +# - create sysroot with init-fs and os-init and syslinux +# sysroot has /ostree basics (but no contents), syslinux cfg and empty root dirs + +echo "1..3" + +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo remote add --set=gpg-verify=false testos $(cat httpd-address)/ostree/testos-repo +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull testos testos/buildmaster/x86_64-runtime +rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) +echo "rev=${rev}" + +# Now sysroot/ostree has the objects from the testos-repo (obtained over http +# and kept in remote "testos"), but there is no deployment + +# This initial deployment gets kicked off with some kernel arguments +${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime +assert_has_dir sysroot/boot/ostree/testos-${bootcsum} + +echo "ok deploy" + +# Create a new branch which we want to migrate to +os_repository_new_commit 1 1 testos/buildmaster/newbranch +# bootcsum now refers to this new commit + +# Create a new commit with an empty tree, which marks the original branch as +# EOL, redirecting to the new one. +mkdir empty +${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --tree=dir=$(pwd)/empty --add-metadata-string "ostree.endoflife=Product discontinued" --add-metadata-string "ostree.endoflife-rebase=testos/buildmaster/newbranch" -b testos/buildmaster/x86_64-runtime -s "EOL redirect to new branch" + +echo "ok new branch" + +# Upgrade existing checkout +${CMD_PREFIX} ostree admin upgrade --os=testos --pull-only +${CMD_PREFIX} ostree admin upgrade --os=testos --deploy-only + +# Check we got redirected to the new branch +assert_file_has_content sysroot/boot/loader/entries/ostree-testos-0.conf "${bootcsum}" +rev=$(${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo rev-parse testos/buildmaster/newbranch) +assert_file_has_content sysroot/ostree/deploy/testos/deploy/${rev}.0/usr/bin/content-iteration "1" + +assert_file_has_content sysroot/ostree/deploy/testos/deploy/${rev}.0.origin "newbranch" + +echo "ok update and redirect" From db00c9591f7425c9098f06b818f6c3b4b7c2f352 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 15 May 2017 17:24:52 -0400 Subject: [PATCH 13/86] bin/cookies: Delete dead tmpfile code in cookie list command This was a copy-paste-o. Closes: #871 Approved by: jlebon --- src/ostree/ot-remote-cookie-util.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index e3ca9eac..b80c825c 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -279,22 +279,11 @@ gboolean ot_list_cookies_at (int dfd, const char *jar_path, GError **error) { #ifdef HAVE_LIBCURL - glnx_fd_close int tempfile_fd = -1; - g_autofree char *tempfile_path = NULL; - g_autofree char *dnbuf = NULL; - const char *dn = NULL; g_autoptr(OtCookieParser) parser = NULL; if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, NULL, error)) return FALSE; - dnbuf = dirname (g_strdup (jar_path)); - dn = dnbuf; - if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, - &tempfile_fd, &tempfile_path, - error)) - return FALSE; - while (ot_parse_cookies_next (parser)) { g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); From e99777e8d23eb59bc6f22cad023fa4d6ce56bbce Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 17 May 2017 11:02:56 -0400 Subject: [PATCH 14/86] Add stub for new libglnx tmpfile API, port simpler callers to it It's hard right now to do a full port to the new libglnx tmpfile API since there are complex cases in the commit path which deal with symlinks as well. Let's make things more gradual by introducing the important part (struct with autocleanup) here in libotutil, port what we can. This will make a future complete port easier. Closes: #871 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 33 ++++++----------- src/libostree/ostree-repo-checkout.c | 40 +++++++++----------- src/libotutil/ot-fs-utils.c | 55 ++++++++++++++++++++++++++++ src/libotutil/ot-fs-utils.h | 24 ++++++++++++ src/ostree/ot-remote-cookie-util.c | 25 +++++-------- 5 files changed, 117 insertions(+), 60 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index f6893fd0..d1dab095 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -99,8 +99,7 @@ struct FetcherRequest { OstreeFetcherRequestFlags flags; gboolean is_membuf; GError *caught_write_error; - char *out_tmpfile; - int out_tmpfile_fd; + OtTmpfile tmpf; GString *output_buf; CURL *easy; @@ -269,12 +268,11 @@ request_get_uri (FetcherRequest *req, guint idx) static gboolean ensure_tmpfile (FetcherRequest *req, GError **error) { - if (req->out_tmpfile_fd == -1) + if (!req->tmpf.initialized) { - if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", - O_WRONLY | O_CLOEXEC, &req->out_tmpfile_fd, - &req->out_tmpfile, - error)) + if (!ot_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", + O_WRONLY | O_CLOEXEC, &req->tmpf, + error)) return FALSE; } return TRUE; @@ -387,18 +385,14 @@ check_multi_info (OstreeFetcher *fetcher) { g_task_return_error (task, g_steal_pointer (&local_error)); } - else if (fchmod (req->out_tmpfile_fd, 0644) < 0) + else if (fchmod (req->tmpf.fd, 0644) < 0) { glnx_set_error_from_errno (error); g_task_return_error (task, g_steal_pointer (&local_error)); } - else if (!glnx_link_tmpfile_at (fetcher->tmpdir_dfd, - GLNX_LINK_TMPFILE_REPLACE, - req->out_tmpfile_fd, - req->out_tmpfile, - fetcher->tmpdir_dfd, - tmpfile_path, - error)) + else if (!ot_link_tmpfile_at (&req->tmpf, GLNX_LINK_TMPFILE_REPLACE, + fetcher->tmpdir_dfd, tmpfile_path, + error)) g_task_return_error (task, g_steal_pointer (&local_error)); else { @@ -586,8 +580,8 @@ write_cb (void *ptr, size_t size, size_t nmemb, void *data) { if (!ensure_tmpfile (req, &req->caught_write_error)) return -1; - g_assert (req->out_tmpfile_fd >= 0); - if (glnx_loop_write (req->out_tmpfile_fd, ptr, realsize) < 0) + g_assert (req->tmpf.fd >= 0); + if (glnx_loop_write (req->tmpf.fd, ptr, realsize) < 0) { glnx_set_error_from_errno (&req->caught_write_error); return -1; @@ -622,9 +616,7 @@ request_unref (FetcherRequest *req) g_ptr_array_unref (req->mirrorlist); g_free (req->filename); g_clear_error (&req->caught_write_error); - if (req->out_tmpfile_fd != -1) - (void) close (req->out_tmpfile_fd); - g_free (req->out_tmpfile); + ot_tmpfile_clear (&req->tmpf); if (req->output_buf) g_string_free (req->output_buf, TRUE); curl_easy_cleanup (req->easy); @@ -847,7 +839,6 @@ _ostree_fetcher_request_async (OstreeFetcher *self, /* We'll allocate the tmpfile on demand, so we handle * file I/O errors just in the write func. */ - req->out_tmpfile_fd = -1; if (req->is_membuf) req->output_buf = g_string_new (""); diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index f6e91498..360c939f 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -60,13 +60,11 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, guint32 file_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode"); file_mode &= ~(S_ISUID|S_ISGID); - glnx_fd_close int fd = -1; - g_autofree char *temp_filename = NULL; - if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY | O_CLOEXEC, - &fd, &temp_filename, - error)) + g_auto(OtTmpfile) tmpf = { 0, }; + if (!ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; - g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (fd, FALSE); + g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (tmpf.fd, FALSE); if (g_output_stream_splice (temp_out, content, 0, cancellable, error) < 0) return FALSE; @@ -76,14 +74,14 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, if (!self->disable_fsync) { - if (TEMP_FAILURE_RETRY (fsync (fd)) < 0) + if (TEMP_FAILURE_RETRY (fsync (tmpf.fd)) < 0) return glnx_throw_errno (error); } if (!g_output_stream_close (temp_out, cancellable, error)) return FALSE; - if (fchmod (fd, file_mode) < 0) + if (fchmod (tmpf.fd, file_mode) < 0) return glnx_throw_errno (error); if (!_ostree_repo_ensure_loose_objdir_at (self->uncompressed_objects_dir_fd, @@ -91,10 +89,9 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, cancellable, error)) return FALSE; - if (!glnx_link_tmpfile_at (self->tmp_dir_fd, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, - fd, temp_filename, - self->uncompressed_objects_dir_fd, loose_path, - error)) + if (!ot_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, + self->uncompressed_objects_dir_fd, loose_path, + error)) return FALSE; return TRUE; @@ -185,7 +182,6 @@ create_file_copy_from_input_at (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - g_autofree char *temp_filename = NULL; const gboolean union_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; const gboolean add_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; const gboolean sepolicy_enabled = options->sepolicy && !repo->disable_xattrs; @@ -252,7 +248,7 @@ create_file_copy_from_input_at (OstreeRepo *repo, } else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { - glnx_fd_close int temp_fd = -1; + g_auto(OtTmpfile) tmpf = { 0, }; guint32 file_mode; GLnxLinkTmpfileReplaceMode replace_mode; @@ -261,9 +257,8 @@ create_file_copy_from_input_at (OstreeRepo *repo, if (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) file_mode &= ~(S_ISUID|S_ISGID); - if (!glnx_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, - &temp_fd, &temp_filename, - error)) + if (!ot_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; if (sepolicy_enabled) @@ -274,11 +269,11 @@ create_file_copy_from_input_at (OstreeRepo *repo, g_file_info_get_attribute_uint32 (file_info, "unix::mode"), &label, cancellable, error)) return FALSE; - if (fsetxattr (temp_fd, "security.selinux", label, strlen (label), 0) < 0) + if (fsetxattr (tmpf.fd, "security.selinux", label, strlen (label), 0) < 0) return glnx_throw_errno_prefix (error, "Setting security.selinux"); } - if (!write_regular_file_content (repo, options, temp_fd, file_info, xattrs, input, + if (!write_regular_file_content (repo, options, tmpf.fd, file_info, xattrs, input, cancellable, error)) return FALSE; @@ -290,10 +285,9 @@ create_file_copy_from_input_at (OstreeRepo *repo, else replace_mode = GLNX_LINK_TMPFILE_NOREPLACE; - if (!glnx_link_tmpfile_at (destination_dfd, replace_mode, - temp_fd, temp_filename, destination_dfd, - destination_name, - error)) + if (!ot_link_tmpfile_at (&tmpf, replace_mode, + destination_dfd, destination_name, + error)) return FALSE; } else diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index 529077fb..fe185e66 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -25,6 +25,61 @@ #include #include +/* Before https://github.com/GNOME/libglnx/commit/9929adc, the libglnx + * tmpfile API made it hard to clean up tmpfiles in failure cases. + * it's API breaking. Carry the fix here until we're ready to fully port. + */ +void +ot_tmpfile_clear (OtTmpfile *tmpf) +{ + if (!tmpf->initialized) + return; + if (tmpf->fd == -1) + return; + (void) close (tmpf->fd); + /* If ->path is set, we're likely aborting due to an error. Clean it up */ + if (tmpf->path) + { + (void) unlinkat (tmpf->src_dfd, tmpf->path, 0); + g_free (tmpf->path); + } +} + +gboolean +ot_open_tmpfile_linkable_at (int dfd, + const char *subpath, + int flags, + OtTmpfile *out_tmpf, + GError **error) +{ + if (!glnx_open_tmpfile_linkable_at (dfd, subpath, flags, &out_tmpf->fd, &out_tmpf->path, error)) + return FALSE; + out_tmpf->initialized = TRUE; + out_tmpf->src_dfd = dfd; + return TRUE; +} + +gboolean +ot_link_tmpfile_at (OtTmpfile *tmpf, + GLnxLinkTmpfileReplaceMode mode, + int target_dfd, + const char *target, + GError **error) +{ + g_return_val_if_fail (tmpf->initialized, FALSE); + glnx_fd_close int fd = glnx_steal_fd (&tmpf->fd); + if (!glnx_link_tmpfile_at (tmpf->src_dfd, mode, fd, tmpf->path, + target_dfd, target, error)) + { + if (tmpf->path) + (void) unlinkat (tmpf->src_dfd, tmpf->path, 0); + tmpf->initialized = FALSE; + return FALSE; + } + tmpf->initialized = FALSE; + return TRUE; +} + /* Convert a fd-relative path to a GFile* - use * for legacy code. */ diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index 14df8acb..26c5a499 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -25,6 +25,30 @@ G_BEGIN_DECLS +/* This is a copy of https://github.com/GNOME/libglnx/pull/46 until we + * can do a full port; see https://github.com/ostreedev/ostree/pull/861 */ +typedef struct { + gboolean initialized; + int src_dfd; + int fd; + char *path; +} OtTmpfile; +void ot_tmpfile_clear (OtTmpfile *tmpf); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtTmpfile, ot_tmpfile_clear); + +gboolean +ot_open_tmpfile_linkable_at (int dfd, + const char *subpath, + int flags, + OtTmpfile *out_tmpf, + GError **error); +gboolean +ot_link_tmpfile_at (OtTmpfile *tmpf, + GLnxLinkTmpfileReplaceMode flags, + int target_dfd, + const char *target, + GError **error); + GFile * ot_fdrel_to_gfile (int dfd, const char *path); gboolean ot_readlinkat_gfile_info (int dfd, diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index b80c825c..9e152e18 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -202,8 +202,7 @@ ot_delete_cookie_at (int dfd, const char *jar_path, { gboolean found = FALSE; #ifdef HAVE_LIBCURL - glnx_fd_close int tempfile_fd = -1; - g_autofree char *tempfile_path = NULL; + g_auto(OtTmpfile) tmpf = { 0, }; g_autofree char *dnbuf = NULL; const char *dn = NULL; g_autoptr(OtCookieParser) parser = NULL; @@ -213,9 +212,8 @@ ot_delete_cookie_at (int dfd, const char *jar_path, dnbuf = g_strdup (jar_path); dn = dirname (dnbuf); - if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, - &tempfile_fd, &tempfile_path, - error)) + if (!ot_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tmpf, error)) return FALSE; while (ot_parse_cookies_next (parser)) @@ -229,19 +227,14 @@ ot_delete_cookie_at (int dfd, const char *jar_path, continue; } - if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || - glnx_loop_write (tempfile_fd, "\n", 1) < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } + if (glnx_loop_write (tmpf.fd, parser->line, strlen (parser->line)) < 0 || + glnx_loop_write (tmpf.fd, "\n", 1) < 0) + return glnx_throw_errno_prefix (error, "write"); } - if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, - tempfile_fd, - tempfile_path, - AT_FDCWD, jar_path, - error)) + if (!ot_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, + AT_FDCWD, jar_path, + error)) return FALSE; #else GSList *cookies; From 56188808b47cd93009816f6ee6e11f56c9356179 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 23 May 2017 13:26:25 +0100 Subject: [PATCH 15/86] build: Use AM_TESTS_ENVIRONMENT rather than TESTS_ENVIRONMENT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TESTS_ENVIRONMENT is reserved for the user to be able to set when running the tests. AM_TESTS_ENVIRONMENT is for the tests’ Makefile to set itself. https://www.gnu.org/software/automake/manual/html_node/Scripts_002dbased-Testsuites.html Signed-off-by: Philip Withnall Closes: #880 Approved by: cgwalters --- Makefile-tests.am | 4 ++-- buildutil/glib-tap.mk | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 6aae5f05..4261fa7c 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -27,7 +27,7 @@ EXTRA_DIST += \ # We should probably consider flipping the default for DEBUG. Also, # include the builddir in $PATH so we find our just-built ostree # binary. -TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ +AM_TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ OSTREE_UNINSTALLED_SRCDIR=$(abs_top_srcdir) \ OSTREE_UNINSTALLED=$(abs_top_builddir) \ G_DEBUG=fatal-warnings \ @@ -37,7 +37,7 @@ TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ OSTREE_FEATURES="$(OSTREE_FEATURES)" \ $(NULL) if BUILDOPT_ASAN -TESTS_ENVIRONMENT += OT_SKIP_READDIR_RAND=1 G_SLICE=always-malloc +AM_TESTS_ENVIRONMENT += OT_SKIP_READDIR_RAND=1 G_SLICE=always-malloc endif uninstalled_test_data = tests/ostree-symlink-stamp tests/ostree-prepare-root-symlink-stamp \ diff --git a/buildutil/glib-tap.mk b/buildutil/glib-tap.mk index ac4329bf..34b8f21e 100644 --- a/buildutil/glib-tap.mk +++ b/buildutil/glib-tap.mk @@ -1,6 +1,6 @@ # GLIB - Library of useful C routines -TESTS_ENVIRONMENT= \ +AM_TESTS_ENVIRONMENT= \ G_TEST_SRCDIR="$(abs_srcdir)" \ G_TEST_BUILDDIR="$(abs_builddir)" \ UNINSTALLEDTESTS=1 \ From be0c02d4f99fc071b18c21856b61c340bb44d5ae Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 24 May 2017 11:12:05 +0200 Subject: [PATCH 16/86] fetcher: Send Accept-Encoding: gzip when downloading summary The summary file can get large, but it compresses well (something which is not true of other files in the ostree repo which are already compressed). By sending Accept-Encoding: gzip (and handling the compressed results) we send a lot less data. I set up the flathub repo (http://flathub.org/repo) to enable gzip for the summary file (only), and the result is that the 331514 byte large summary was transferred in 122889 bytes. On my (fast) network this decreased the time i took to do "flatpak remote-ls flathub" by about 100msec. This fixes https://github.com/ostreedev/ostree/issues/802 Closes: #882 Approved by: cgwalters --- src/libostree/ostree-fetcher-curl.c | 3 +++ src/libostree/ostree-fetcher-soup.c | 5 +++++ src/libostree/ostree-fetcher.h | 3 ++- src/libostree/ostree-repo-pull.c | 8 ++++++-- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index d1dab095..f483a6bb 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -762,6 +762,9 @@ initiate_next_curl_request (FetcherRequest *req, curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path); } + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) > 0) + curl_easy_setopt (req->easy, CURLOPT_ACCEPT_ENCODING, ""); + /* We should only speak HTTP; TODO: only enable file if specified */ curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE)); /* Picked the current version in F25 as of 20170127, since diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index fdcbea52..1ca2e771 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -63,6 +63,7 @@ typedef struct { GVariant *extra_headers; int max_outstanding; + gboolean transfer_gzip; /* Our active HTTP requests */ GHashTable *outstanding; @@ -553,6 +554,9 @@ ostree_fetcher_session_thread (gpointer data) SOUP_SESSION_IDLE_TIMEOUT, 60, NULL); + if (closure->transfer_gzip) + soup_session_add_feature_by_type (closure->session, SOUP_TYPE_CONTENT_DECODER); + /* XXX: Now that we have mirrorlist support, we could make this even smarter * by spreading requests across mirrors. */ g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL); @@ -663,6 +667,7 @@ _ostree_fetcher_constructed (GObject *object) self->thread_closure->ref_count = 1; self->thread_closure->main_context = g_main_context_ref (main_context); self->thread_closure->running = 1; + self->thread_closure->transfer_gzip = (self->config_flags & OSTREE_FETCHER_FLAGS_TRANSFER_GZIP) != 0; self->thread_closure->tmpdir_dfd = -1; self->thread_closure->tmpdir_lock = empty_lockfile; diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 78b29fae..16cf3d7d 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -48,7 +48,8 @@ struct OstreeFetcherClass typedef enum { OSTREE_FETCHER_FLAGS_NONE = 0, - OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0) + OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0), + OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1) } OstreeFetcherConfigFlags; void diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 08692c59..751129e6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2161,6 +2161,7 @@ _ostree_repo_cache_summary (OstreeRepo *self, static OstreeFetcher * _ostree_repo_remote_new_fetcher (OstreeRepo *self, const char *remote_name, + gboolean gzip, GError **error) { OstreeFetcher *fetcher = NULL; @@ -2179,6 +2180,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, if (tls_permissive) fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE; + if (gzip) + fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP; + fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags); { @@ -2436,7 +2440,7 @@ repo_remote_fetch_summary (OstreeRepo *self, mainctx = g_main_context_new (); g_main_context_push_thread_default (mainctx); - fetcher = _ostree_repo_remote_new_fetcher (self, name, error); + fetcher = _ostree_repo_remote_new_fetcher (self, name, TRUE, error); if (fetcher == NULL) goto out; @@ -2544,7 +2548,7 @@ static gboolean reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **error) { g_clear_object (&pull_data->fetcher); - pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, error); + pull_data->fetcher = _ostree_repo_remote_new_fetcher (pull_data->repo, remote_name, FALSE, error); if (pull_data->fetcher == NULL) return FALSE; From c6960e63b2c66b5e1523427173e1b48c14aa69c2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Mar 2017 17:18:27 -0400 Subject: [PATCH 17/86] lib/deploy: Port config merge logic to new code style This is a de-scoping of work I did in preparation for rpm-ostree [live updates](https://github.com/projectatomic/rpm-ostree/pull/652). Originally I was going to expose this as a public API. However, I decided to do things differently, but the cleanup here for new code style and fd-relative is nice to have anyways. We rework things to use `OstreeDeployment*`, which the caller is expected to already have, rather than `GFile*`s pointing to the config directories. Closes: #741 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 158 +++++++++++++------------- 1 file changed, 76 insertions(+), 82 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 257a058b..6df4fff2 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -423,11 +423,18 @@ copy_modified_config_file (int orig_etc_fd, return ret; } -/** - * merge_etc_changes: +/* + * merge_configuration_from: + * @sysroot: Sysroot + * @merge_deployment: Source of configuration differences + * @merge_deployment_dfd: Directory fd, may be -1 + * @new_deployment: Target for merge of configuration + * @new_deployment_dfd: Directory fd for @new_deployment (may *not* be -1) + * @cancellable: Cancellable + * @error: Error * - * Compute the difference between @orig_etc and @modified_etc, - * and apply that to @new_etc. + * Compute the difference between @merge_deployment's `/usr/etc` and `/etc`, and + * apply that to @new_deployment's `/etc`. * * The algorithm for computing the difference is pretty simple; it's * approximately equivalent to "diff -unR orig_etc modified_etc", @@ -435,26 +442,37 @@ copy_modified_config_file (int orig_etc_fd, * changed in @new_etc, the modified version always wins. */ static gboolean -merge_etc_changes (GFile *orig_etc, - GFile *modified_etc, - GFile *new_etc, - OstreeSysrootDebugFlags flags, - GCancellable *cancellable, - GError **error) +merge_configuration_from (OstreeSysroot *sysroot, + OstreeDeployment *merge_deployment, + int merge_deployment_dfd, + OstreeDeployment *new_deployment, + int new_deployment_dfd, + GCancellable *cancellable, + GError **error) { - gboolean ret = FALSE; - g_autoptr(GPtrArray) modified = NULL; - g_autoptr(GPtrArray) removed = NULL; - g_autoptr(GPtrArray) added = NULL; - guint i; - glnx_fd_close int orig_etc_fd = -1; - glnx_fd_close int modified_etc_fd = -1; - glnx_fd_close int new_etc_fd = -1; + glnx_fd_close int owned_merge_deployment_dfd = -1; + const OstreeSysrootDebugFlags flags = sysroot->debug_flags; - modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref); - removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); - added = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); + g_assert (merge_deployment != NULL && new_deployment != NULL); + g_assert (new_deployment_dfd != -1); + /* Allow the caller to pass -1 for the merge, for convenience */ + if (merge_deployment_dfd == -1) + { + g_autofree char *merge_deployment_path = ostree_sysroot_get_deployment_dirpath (sysroot, merge_deployment); + if (!glnx_opendirat (sysroot->sysroot_fd, merge_deployment_path, FALSE, + &owned_merge_deployment_dfd, error)) + return FALSE; + merge_deployment_dfd = owned_merge_deployment_dfd; + } + + /* TODO: get rid of GFile usage here */ + g_autoptr(GFile) orig_etc = ot_fdrel_to_gfile (merge_deployment_dfd, "usr/etc"); + g_autoptr(GFile) modified_etc = ot_fdrel_to_gfile (merge_deployment_dfd, "etc"); + /* Return values for below */ + g_autoptr(GPtrArray) modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref); + g_autoptr(GPtrArray) removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); + g_autoptr(GPtrArray) added = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); /* For now, ignore changes to xattrs; the problem is that * security.selinux will be different between the /usr/etc labels * and the ones in the real /etc, so they all show up as different. @@ -466,42 +484,37 @@ merge_etc_changes (GFile *orig_etc, if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_IGNORE_XATTRS, orig_etc, modified_etc, modified, removed, added, cancellable, error)) - { - g_prefix_error (error, "While computing configuration diff: "); - goto out; - } + return glnx_prefix_error (error, "While computing configuration diff"); ot_log_structured_print_id_v (OSTREE_CONFIGMERGE_ID, - "Copying /etc changes: %u modified, %u removed, %u added", + "Copying /etc changes: %u modified, %u removed, %u added", modified->len, removed->len, added->len); - if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (orig_etc), TRUE, - &orig_etc_fd, error)) - goto out; - if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (modified_etc), TRUE, - &modified_etc_fd, error)) - goto out; - if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (new_etc), TRUE, - &new_etc_fd, error)) - goto out; + glnx_fd_close int orig_etc_fd = -1; + if (!glnx_opendirat (merge_deployment_dfd, "usr/etc", TRUE, &orig_etc_fd, error)) + return FALSE; + glnx_fd_close int modified_etc_fd = -1; + if (!glnx_opendirat (merge_deployment_dfd, "etc", TRUE, &modified_etc_fd, error)) + return FALSE; + glnx_fd_close int new_etc_fd = -1; + if (!glnx_opendirat (new_deployment_dfd, "etc", TRUE, &new_etc_fd, error)) + return FALSE; - for (i = 0; i < removed->len; i++) + for (guint i = 0; i < removed->len; i++) { GFile *file = removed->pdata[i]; - g_autoptr(GFile) target_file = NULL; g_autofree char *path = NULL; path = g_file_get_relative_path (orig_etc, file); g_assert (path); - target_file = g_file_resolve_relative_path (new_etc, path); - if (!glnx_shutil_rm_rf_at (AT_FDCWD, gs_file_get_path_cached (target_file), cancellable, error)) - goto out; + if (!glnx_shutil_rm_rf_at (new_etc_fd, path, cancellable, error)) + return FALSE; } - for (i = 0; i < modified->len; i++) + for (guint i = 0; i < modified->len; i++) { OstreeDiffItem *diff = modified->pdata[i]; g_autofree char *path = g_file_get_relative_path (modified_etc, diff->target); @@ -510,9 +523,9 @@ merge_etc_changes (GFile *orig_etc, if (!copy_modified_config_file (orig_etc_fd, modified_etc_fd, new_etc_fd, path, flags, cancellable, error)) - goto out; + return FALSE; } - for (i = 0; i < added->len; i++) + for (guint i = 0; i < added->len; i++) { GFile *file = added->pdata[i]; g_autofree char *path = g_file_get_relative_path (modified_etc, file); @@ -521,12 +534,10 @@ merge_etc_changes (GFile *orig_etc, if (!copy_modified_config_file (orig_etc_fd, modified_etc_fd, new_etc_fd, path, flags, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -775,16 +786,7 @@ merge_configuration (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *deployment_abspath = glnx_fdrel_abspath (deployment_dfd, "."); - g_autoptr(GFile) deployment_path = g_file_new_for_path (deployment_abspath); - g_autoptr(GFile) source_etc_path = NULL; - g_autoptr(GFile) source_etc_pristine_path = NULL; - g_autoptr(GFile) deployment_usretc_path = NULL; - g_autoptr(GFile) deployment_etc_path = NULL; glnx_unref_object OstreeSePolicy *sepolicy = NULL; - gboolean etc_exists; - gboolean usretc_exists; if (previous_deployment) { @@ -792,8 +794,6 @@ merge_configuration (OstreeSysroot *sysroot, OstreeBootconfigParser *previous_bootconfig; previous_path = ostree_sysroot_get_deployment_directory (sysroot, previous_deployment); - source_etc_path = g_file_resolve_relative_path (previous_path, "etc"); - source_etc_pristine_path = g_file_resolve_relative_path (previous_path, "usr/etc"); previous_bootconfig = ostree_deployment_get_bootconfig (previous_deployment); if (previous_bootconfig) @@ -807,26 +807,20 @@ merge_configuration (OstreeSysroot *sysroot, } } - deployment_etc_path = g_file_get_child (deployment_path, "etc"); - deployment_usretc_path = g_file_resolve_relative_path (deployment_path, "usr/etc"); - - etc_exists = g_file_query_exists (deployment_etc_path, NULL); - usretc_exists = g_file_query_exists (deployment_usretc_path, NULL); + gboolean etc_exists = FALSE; + if (!ot_query_exists_at (deployment_dfd, "etc", &etc_exists, error)) + return FALSE; + gboolean usretc_exists = FALSE; + if (!ot_query_exists_at (deployment_dfd, "usr/etc", &usretc_exists, error)) + return FALSE; if (etc_exists && usretc_exists) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Tree contains both /etc and /usr/etc"); - goto out; - } + return glnx_throw (error, "Tree contains both /etc and /usr/etc"); else if (etc_exists) { /* Compatibility hack */ if (renameat (deployment_dfd, "etc", deployment_dfd, "usr/etc") < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "renameat"); usretc_exists = TRUE; etc_exists = FALSE; } @@ -843,9 +837,9 @@ merge_configuration (OstreeSysroot *sysroot, /* Here, we initialize SELinux policy from the /usr/etc inside * the root - this is before we've finalized the configuration * merge into /etc. */ - sepolicy = ostree_sepolicy_new (deployment_path, cancellable, error); + sepolicy = ostree_sepolicy_new_at (deployment_dfd, cancellable, error); if (!sepolicy) - goto out; + return FALSE; if (ostree_sepolicy_get_name (sepolicy) != NULL) etc_co_opts.sepolicy = sepolicy; @@ -854,21 +848,21 @@ merge_configuration (OstreeSysroot *sysroot, deployment_dfd, "etc", ostree_deployment_get_csum (deployment), cancellable, error)) - goto out; + return FALSE; + } - if (source_etc_path) + if (previous_deployment) { - if (!merge_etc_changes (source_etc_pristine_path, source_etc_path, deployment_etc_path, - sysroot->debug_flags, cancellable, error)) - goto out; + if (!merge_configuration_from (sysroot, previous_deployment, -1, + deployment, deployment_dfd, + cancellable, error)) + return FALSE; } - ret = TRUE; if (out_sepolicy) *out_sepolicy = g_steal_pointer (&sepolicy); - out: - return ret; + return TRUE; } static gboolean From 822ade62c61e5940452df6fbb855f79d51864dc6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 24 May 2017 16:43:39 -0400 Subject: [PATCH 18/86] tests: Add some C tests for object writing Prep for https://github.com/ostreedev/ostree/pull/881 Closes: #884 Approved by: jlebon --- tests/test-basic-c.c | 51 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/test-basic-c.c b/tests/test-basic-c.c index 8e7882b9..dbab087e 100644 --- a/tests/test-basic-c.c +++ b/tests/test-basic-c.c @@ -170,6 +170,54 @@ test_raw_file_to_archive_z2_stream (gconstpointer data) g_assert_cmpint (checks, >, 0); } +static gboolean hi_content_stream_new (GInputStream **out_stream, + guint64 *out_length, + GError **error) +{ + static const char hi[] = "hi"; + g_autoptr(GMemoryInputStream) hi_memstream = (GMemoryInputStream*)g_memory_input_stream_new_from_data (hi, sizeof(hi)-1, NULL); + g_autoptr(GFileInfo) finfo = g_file_info_new (); + g_file_info_set_attribute_uint32 (finfo, "standard::type", G_FILE_TYPE_REGULAR); + g_file_info_set_attribute_boolean (finfo, "standard::is-symlink", FALSE); + g_file_info_set_attribute_uint32 (finfo, "unix::uid", 0); + g_file_info_set_attribute_uint32 (finfo, "unix::gid", 0); + g_file_info_set_attribute_uint32 (finfo, "unix::mode", S_IFREG|0644); + return ostree_raw_file_to_content_stream ((GInputStream*)hi_memstream, finfo, NULL, out_stream, out_length, NULL, error); +} + +static void +test_object_writes (gconstpointer data) +{ + OstreeRepo *repo = OSTREE_REPO (data); + g_autoptr(GError) error = NULL; + + static const char hi_sha256[] = "2301b5923720c3edc1f0467addb5c287fd5559e3e0cd1396e7f1edb6b01be9f0"; + + /* Successful content write */ + { g_autoptr(GInputStream) hi_memstream = NULL; + guint64 len; + hi_content_stream_new (&hi_memstream, &len, &error); + g_assert_no_error (error); + g_autofree guchar *csum = NULL; + (void)ostree_repo_write_content (repo, hi_sha256, hi_memstream, len, &csum, + NULL, &error); + g_assert_no_error (error); + } + + /* Invalid content write */ + { g_autoptr(GInputStream) hi_memstream = NULL; + guint64 len; + hi_content_stream_new (&hi_memstream, &len, &error); + g_assert_no_error (error); + g_autofree guchar *csum = NULL; + static const char invalid_hi_sha256[] = "cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe"; + g_assert (!ostree_repo_write_content (repo, invalid_hi_sha256, hi_memstream, len, &csum, + NULL, &error)); + g_assert (error); + g_assert (strstr (error->message, "Corrupted file object")); + } +} + int main (int argc, char **argv) { g_autoptr(GError) error = NULL; @@ -180,9 +228,10 @@ int main (int argc, char **argv) repo = ot_test_setup_repo (NULL, &error); if (!repo) goto out; - + g_test_add_data_func ("/repo-not-system", repo, test_repo_is_not_system); g_test_add_data_func ("/raw-file-to-archive-z2-stream", repo, test_raw_file_to_archive_z2_stream); + g_test_add_data_func ("/objectwrites", repo, test_object_writes); return g_test_run(); out: From 07acb5b82cfab8383a89c58e3197bcbc8fcb0d0f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 25 May 2017 12:28:15 -0400 Subject: [PATCH 19/86] PAPR: migrate to the new name The redhat-ci service has been renamed to PAPR. Previous values are still supported though should be considered deprecated. Closes: #885 Approved by: cgwalters --- .redhat-ci.Dockerfile => .papr.Dockerfile | 0 .redhat-ci.yml => .papr.yml | 0 tests/ci-commitmessage-submodules.sh | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename .redhat-ci.Dockerfile => .papr.Dockerfile (100%) rename .redhat-ci.yml => .papr.yml (100%) diff --git a/.redhat-ci.Dockerfile b/.papr.Dockerfile similarity index 100% rename from .redhat-ci.Dockerfile rename to .papr.Dockerfile diff --git a/.redhat-ci.yml b/.papr.yml similarity index 100% rename from .redhat-ci.yml rename to .papr.yml diff --git a/tests/ci-commitmessage-submodules.sh b/tests/ci-commitmessage-submodules.sh index 8ce077c3..aeccc24d 100755 --- a/tests/ci-commitmessage-submodules.sh +++ b/tests/ci-commitmessage-submodules.sh @@ -13,9 +13,9 @@ set -euo pipefail # It's very common for people to accidentally change submodules, and having this # requirement is a small hurdle to pass. -# if running under RHCI, use the branch/PR HEAD actually +# if running under PAPR, use the branch/PR HEAD actually # being tested rather than the merge sha -HEAD=${RHCI_COMMIT:-HEAD} +HEAD=${PAPR_COMMIT:-HEAD} tmpd=$(mktemp -d) touch ${tmpd}/.tmpdir From ad119aece9a6c9c547d6a8dbbac020d1c0e4fdad Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 May 2017 10:22:52 -0400 Subject: [PATCH 20/86] pull-test: Add some 404 tests See: https://github.com/flatpak/flatpak/issues/816 Closes: #887 Approved by: jlebon --- tests/pull-test.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 3a836da9..4a4ef069 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..18" +echo "1..20" # Try both syntaxes repo_init --no-gpg-verify @@ -295,6 +295,23 @@ assert_file_has_content err.txt "ONE BILLION DOLLARS" echo "ok unconfigured" +cd ${test_tmpdir} +repo_init +${CMD_PREFIX} ostree --repo=repo remote add origin-bad $(cat httpd-address)/ostree/noent +if ${CMD_PREFIX} ostree --repo=repo --depth=0 pull origin-bad main 2>err.txt; then + assert_not_reached "pull repo 404 succeeded?" +fi +assert_file_has_content err.txt "404" +echo "ok pull repo 404" + +cd ${test_tmpdir} +repo_init --set=gpg-verify=true +if ${CMD_PREFIX} ostree --repo=repo --depth=0 pull origin main 2>err.txt; then + assert_not_reached "pull repo 404 succeeded?" +fi +assert_file_has_content err.txt "GPG verification enabled, but no signatures found" +echo "ok pull repo 404 (gpg)" + cd ${test_tmpdir} repo_init --set=gpg-verify=true ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit \ From ff2b881275f641bd934207d58ba14289a4190d0e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 May 2017 13:40:14 -0400 Subject: [PATCH 21/86] lib/fsutil: Delete unused GFile ioctl method All the deployment code uses fds. Closes: #889 Approved by: jlebon --- src/libostree/ostree-linuxfsutil.c | 32 ------------------------------ src/libostree/ostree-linuxfsutil.h | 6 ------ 2 files changed, 38 deletions(-) diff --git a/src/libostree/ostree-linuxfsutil.c b/src/libostree/ostree-linuxfsutil.c index b270a94e..dd717118 100644 --- a/src/libostree/ostree-linuxfsutil.c +++ b/src/libostree/ostree-linuxfsutil.c @@ -99,35 +99,3 @@ _ostree_linuxfs_fd_alter_immutable_flag (int fd, out: return ret; } - -gboolean -_ostree_linuxfs_alter_immutable_flag (GFile *path, - gboolean new_immutable_state, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - glnx_fd_close int fd = -1; - - if (g_cancellable_set_error_if_cancelled (cancellable, error)) - return FALSE; - - fd = open (gs_file_get_path_cached (path), O_RDONLY|O_NONBLOCK|O_LARGEFILE); - if (fd == -1) - { - int errsv = errno; - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "open(%s): %s", - gs_file_get_path_cached (path), - g_strerror (errsv)); - goto out; - } - - if (!_ostree_linuxfs_fd_alter_immutable_flag (fd, new_immutable_state, - cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} diff --git a/src/libostree/ostree-linuxfsutil.h b/src/libostree/ostree-linuxfsutil.h index 9c74b8a9..84122a70 100644 --- a/src/libostree/ostree-linuxfsutil.h +++ b/src/libostree/ostree-linuxfsutil.h @@ -30,10 +30,4 @@ _ostree_linuxfs_fd_alter_immutable_flag (int fd, GCancellable *cancellable, GError **error); -gboolean -_ostree_linuxfs_alter_immutable_flag (GFile *path, - gboolean new_immutable_state, - GCancellable *cancellable, - GError **error); - G_END_DECLS From 2f834968c6b0602e25cbdc4d7c6f94d3f57a3f8b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 May 2017 13:41:12 -0400 Subject: [PATCH 22/86] lib/fsutil: Port to new code style Pretty trivial. Closes: #889 Approved by: jlebon --- src/libostree/ostree-linuxfsutil.c | 36 +++++++++--------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-linuxfsutil.c b/src/libostree/ostree-linuxfsutil.c index dd717118..9785ac57 100644 --- a/src/libostree/ostree-linuxfsutil.c +++ b/src/libostree/ostree-linuxfsutil.c @@ -27,6 +27,8 @@ #include #include +#include "otutil.h" + /** * _ostree_linuxfs_fd_alter_immutable_flag: * @fd: A file descriptor @@ -47,29 +49,21 @@ _ostree_linuxfs_fd_alter_immutable_flag (int fd, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - unsigned long flags; - int r; static gint no_alter_immutable = 0; if (g_atomic_int_get (&no_alter_immutable)) return TRUE; - r = ioctl (fd, EXT2_IOC_GETFLAGS, &flags); + unsigned long flags; + int r = ioctl (fd, EXT2_IOC_GETFLAGS, &flags); if (r == -1) { - int errsv = errno; - if (errsv == EPERM) + if (errno == EPERM) g_atomic_int_set (&no_alter_immutable, 1); - else if (errsv == EOPNOTSUPP || errsv == ENOTTY) + else if (errno == EOPNOTSUPP || errno == ENOTTY) ; else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "ioctl(EXT2_IOC_GETFLAGS): %s", - g_strerror (errsv)); - goto out; - } + return glnx_throw_errno_prefix (error, "ioctl(EXT2_IOC_GETFLAGS)"); } else { @@ -80,22 +74,14 @@ _ostree_linuxfs_fd_alter_immutable_flag (int fd, r = ioctl (fd, EXT2_IOC_SETFLAGS, &flags); if (r == -1) { - int errsv = errno; - if (errsv == EPERM) + if (errno == EPERM) g_atomic_int_set (&no_alter_immutable, 1); - else if (errsv == EOPNOTSUPP || errsv == ENOTTY) + else if (errno == EOPNOTSUPP || errno == ENOTTY) ; else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "ioctl(EXT2_IOC_GETFLAGS): %s", - g_strerror (errsv)); - goto out; - } + return glnx_throw_errno_prefix (error, "ioctl(EXT2_IOC_SETFLAGS)"); } } - ret = TRUE; - out: - return ret; + return TRUE; } From ed430b45de6e298ee573dbeccd88d75057a41c6d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 25 May 2017 17:26:42 -0400 Subject: [PATCH 23/86] lib: Add an "is_system" member to OstreeRepo This is prep for introducing a fd-relative `ostree_repo_new_at()`. Previously, `ostree_repo_is_system()` compared `GFile` paths, but there's a much simpler check we can do first - if this repository was created via `OstreeSysroot`, it must be a system repo. Closes: #886 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 6 ++++++ src/libostree/ostree-sysroot.c | 2 ++ 3 files changed, 9 insertions(+) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 7efbf91f..015a9a8b 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -106,6 +106,7 @@ struct OstreeRepo { gboolean inited; gboolean writable; + gboolean is_system; /* Was this repo created via ostree_sysroot_get_repo() ? */ GError *writable_error; gboolean in_transaction; gboolean disable_fsync; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 7ee2c9aa..cbbaec9b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -710,6 +710,12 @@ ostree_repo_is_system (OstreeRepo *repo) g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); + /* If we were created via ostree_sysroot_get_repo(), we know the answer is yes + * without having to compare file paths. + */ + if (repo->is_system) + return TRUE; + default_repo_path = get_default_repo_path (repo->sysroot_dir); return g_file_equal (repo->repodir, default_repo_path); diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index e47214c5..ac24b0af 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -26,6 +26,7 @@ #include #include "ostree-core-private.h" +#include "ostree-repo-private.h" #include "ostree-sepolicy-private.h" #include "ostree-sysroot-private.h" #include "ostree-deployment-private.h" @@ -133,6 +134,7 @@ ostree_sysroot_constructed (GObject *object) repo_path = g_file_resolve_relative_path (self->path, "ostree/repo"); self->repo = ostree_repo_new_for_sysroot_path (repo_path, self->path); + self->repo->is_system = TRUE; G_OBJECT_CLASS (ostree_sysroot_parent_class)->constructed (object); } From 9bf8a8503afa5cc7acacc8203490330292da7f85 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 25 May 2017 19:38:52 -0400 Subject: [PATCH 24/86] lib/sysroot: Add non-failable ostree_sysroot_repo() Having a failable accessor is annoying, since it's really common to reference both. Instead, open the repo once when we load the sysroot, and provide a non-failable accessor. This is also prep for `ostree_repo_open_at()`, which collapses the separation between `ostree_repo_new()` and `ostree_repo_open()`. Closes: #886 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 6 ++-- src/libostree/ostree-sysroot-cleanup.c | 6 +--- src/libostree/ostree-sysroot-deploy.c | 5 +--- src/libostree/ostree-sysroot-private.h | 1 + src/libostree/ostree-sysroot.c | 40 ++++++++++++++++++++++++-- src/libostree/ostree-sysroot.h | 3 ++ 7 files changed, 47 insertions(+), 15 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index adc2dfd7..56c5c94e 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -486,6 +486,7 @@ ostree_sysroot_get_deployment_dirpath ostree_sysroot_get_deployment_origin_path ostree_sysroot_cleanup ostree_sysroot_prepare_cleanup +ostree_sysroot_repo ostree_sysroot_get_repo ostree_sysroot_init_osname ostree_sysroot_deployment_set_kargs diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index e2a7c3d9..612eb8ac 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -402,12 +402,10 @@ global: * NOTE NOTE NOTE */ -/* -LIBOSTREE_2017.$NEWVERSION { +LIBOSTREE_2017.7 { global: - someostree_symbol_deleteme; + ostree_sysroot_repo; } LIBOSTREE_2017.6; -*/ /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 499f20eb..c0b8bf07 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -501,8 +501,6 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - glnx_unref_object OstreeRepo *repo = NULL; - g_return_val_if_fail (OSTREE_IS_SYSROOT (self), FALSE); g_return_val_if_fail (self->loaded, FALSE); @@ -512,9 +510,7 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, if (!cleanup_old_deployments (self, cancellable, error)) return FALSE; - if (!ostree_sysroot_get_repo (self, &repo, cancellable, error)) - return FALSE; - + OstreeRepo *repo = ostree_sysroot_repo (self); if (!generate_deployment_refs (self, repo, self->bootversion, self->subbootversion, diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 6df4fff2..2b45e6da 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1956,10 +1956,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, if (!glnx_opendirat (self->sysroot_fd, osdeploypath, TRUE, &os_deploy_dfd, error)) return FALSE; - glnx_unref_object OstreeRepo *repo = NULL; - if (!ostree_sysroot_get_repo (self, &repo, cancellable, error)) - return FALSE; - + OstreeRepo *repo = ostree_sysroot_repo (self); glnx_unref_object OstreeDeployment *merge_deployment = NULL; if (provided_merge_deployment != NULL) merge_deployment = g_object_ref (provided_merge_deployment); diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 26cfd363..14ee5cad 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -57,6 +57,7 @@ struct OstreeSysroot { /* Only access through ostree_sysroot_get_repo() */ OstreeRepo *repo; + gboolean repo_opened; OstreeSysrootDebugFlags debug_flags; }; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index ac24b0af..86aa8ce3 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -729,6 +729,18 @@ ostree_sysroot_load (OstreeSysroot *self, return ostree_sysroot_load_if_changed (self, NULL, cancellable, error); } +static gboolean +ensure_repo_opened (OstreeSysroot *self, + GError **error) +{ + if (self->repo_opened) + return TRUE; + if (!ostree_repo_open (self->repo, NULL, error)) + return FALSE; + self->repo_opened = TRUE; + return TRUE; +} + gboolean ostree_sysroot_load_if_changed (OstreeSysroot *self, gboolean *out_changed, @@ -738,6 +750,13 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!ensure_sysroot_fd (self, error)) return FALSE; + /* Here we also lazily initialize the repository. We didn't do this + * previous to v2017.6, but we do now to support the error-free + * ostree_sysroot_repo() API. + */ + if (!ensure_repo_opened (self, error)) + return FALSE; + int bootversion = 0; if (!read_current_bootversion (self, &bootversion, cancellable, error)) return FALSE; @@ -918,8 +937,7 @@ ostree_sysroot_get_repo (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - /* ostree_repo_open() is idempotent. */ - if (!ostree_repo_open (self->repo, cancellable, error)) + if (!ensure_repo_opened (self, error)) return FALSE; if (out_repo != NULL) @@ -927,6 +945,24 @@ ostree_sysroot_get_repo (OstreeSysroot *self, return TRUE; } +/** + * ostree_sysroot_repo: + * @self: Sysroot + * + * This function is a variant of ostree_sysroot_get_repo() that cannot fail, and + * returns a cached repository. Can only be called after ostree_sysroot_load() + * has been invoked successfully. + * + * Returns: (transfer none): The OSTree repository in sysroot @self. + */ +OstreeRepo * +ostree_sysroot_repo (OstreeSysroot *self) +{ + g_return_val_if_fail (self->loaded, NULL); + g_assert (self->repo); + return self->repo; +} + /** * ostree_sysroot_query_bootloader: * @sysroot: Sysroot diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index 09614b55..e5969e9e 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -126,6 +126,9 @@ gboolean ostree_sysroot_write_origin_file (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +OstreeRepo * ostree_sysroot_repo (OstreeSysroot *self); + _OSTREE_PUBLIC gboolean ostree_sysroot_get_repo (OstreeSysroot *self, OstreeRepo **out_repo, From 88a1fc92a9937fafd195d70abc80a30be64fc136 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 May 2017 14:35:34 -0400 Subject: [PATCH 25/86] tree-wide: Add+run spatch to use glnx_throw() I had to run a sed job to add whitespace after, but otherwise this was easy. Closes: #890 Approved by: jlebon --- coccinelle/new-error.cocci | 19 +++++++++++++++++++ src/libostree/ostree-core.c | 10 +++------- src/libostree/ostree-gpg-verify-result.c | 5 ++--- src/libostree/ostree-repo-libarchive.c | 5 ++--- src/libostree/ostree-repo-static-delta-core.c | 5 ++--- .../ostree-repo-static-delta-processing.c | 4 +--- src/libostree/ostree-sysroot-deploy.c | 6 +++--- src/libotutil/ot-tool-util.c | 8 ++------ src/libotutil/ot-unix-utils.c | 12 +++--------- src/ostree/ot-builtin-commit.c | 4 +--- src/ostree/ot-builtin-config.c | 5 ++--- 11 files changed, 40 insertions(+), 43 deletions(-) create mode 100644 coccinelle/new-error.cocci diff --git a/coccinelle/new-error.cocci b/coccinelle/new-error.cocci new file mode 100644 index 00000000..147a8e02 --- /dev/null +++ b/coccinelle/new-error.cocci @@ -0,0 +1,19 @@ +// Conversion for G_IO_ERROR_FAILED that could be glnx_throw() +@@ +expression p; +@@ +- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, p); +- return FALSE; ++ return glnx_throw (error, "%s", p); +@@ +expression p; +@@ +- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, p); +- return FALSE; ++ return glnx_throw (error, p); +@@ +expression p, q; +@@ +- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, p, q); +- return FALSE; ++ return glnx_throw (error, p, q); diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 7d2ed3c6..54e01bcb 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1763,16 +1763,12 @@ validate_variant (GVariant *variant, { if (!g_variant_is_normal_form (variant)) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not normal form"); - return FALSE; + return glnx_throw (error, "%s", "Not normal form"); } if (!g_variant_is_of_type (variant, variant_type)) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Doesn't match variant type '%s'", - (char*)variant_type); - return FALSE; + return glnx_throw (error, "Doesn't match variant type '%s'", + (char *)variant_type); } return TRUE; } diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index cd709b7c..0277ce1e 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -665,9 +665,8 @@ ostree_gpg_verify_result_require_valid_signature (OstreeGpgVerifyResult *result, if (ostree_gpg_verify_result_count_valid (result) == 0) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "GPG signatures found, but none are in trusted keyring"); - return FALSE; + return glnx_throw (error, "%s", + "GPG signatures found, but none are in trusted keyring"); } return TRUE; diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index 01982894..02f1364e 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -640,9 +640,8 @@ aic_handle_entry (OstreeRepoArchiveImportContext *ctx, return TRUE; else { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unsupported file type for path \"%s\"", path); - return FALSE; + return glnx_throw (error, "Unsupported file type for path \"%s\"", + path); } } } diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 3a14f8bb..d83b7f0f 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -45,9 +45,8 @@ _ostree_static_delta_parse_checksum_array (GVariant *array, if (G_UNLIKELY(n > (G_MAXUINT32/OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) || (n_checksums * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) != n)) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid checksum array length %" G_GSIZE_FORMAT, n); - return FALSE; + return glnx_throw (error, + "Invalid checksum array length %" G_GSIZE_FORMAT, n); } *out_checksums_array = (gpointer)g_variant_get_data (array); diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index a71e070f..2280dec9 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -115,9 +115,7 @@ read_varuint64 (StaticDeltaExecutionState *state, gsize bytes_read; if (!_ostree_read_varuint64 (state->opdata, state->oplen, out_value, &bytes_read)) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unexpected EOF reading varint"); - return FALSE; + return glnx_throw (error, "%s", "Unexpected EOF reading varint"); } state->opdata += bytes_read; state->oplen -= bytes_read; diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 2b45e6da..ed4831c1 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1028,9 +1028,9 @@ checksum_from_kernel_src (const char *name, const char *last_dash = strrchr (name, '-'); if (!last_dash) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Malformed kernel/initramfs name '%s', missing '-'", name); - return FALSE; + return glnx_throw (error, + "Malformed kernel/initramfs name '%s', missing '-'", + name); } *out_checksum = g_strdup (last_dash + 1); return TRUE; diff --git a/src/libotutil/ot-tool-util.c b/src/libotutil/ot-tool-util.c index 1043f55f..0743c528 100644 --- a/src/libotutil/ot-tool-util.c +++ b/src/libotutil/ot-tool-util.c @@ -40,9 +40,7 @@ ot_parse_boolean (const char *value, *out_parsed = FALSE; else { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid boolean argument '%s'", value); - return FALSE; + return glnx_throw (error, "Invalid boolean argument '%s'", value); } return TRUE; @@ -57,9 +55,7 @@ ot_parse_keyvalue (const char *keyvalue, const char *eq = strchr (keyvalue, '='); if (!eq) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Missing '=' in KEY=VALUE for --set"); - return FALSE; + return glnx_throw (error, "Missing '=' in KEY=VALUE for --set"); } *out_key = g_strndup (keyvalue, eq - keyvalue); *out_value = g_strdup (eq + 1); diff --git a/src/libotutil/ot-unix-utils.c b/src/libotutil/ot-unix-utils.c index 46dd346e..54547dd9 100644 --- a/src/libotutil/ot-unix-utils.c +++ b/src/libotutil/ot-unix-utils.c @@ -41,21 +41,15 @@ ot_util_filename_validate (const char *name, { if (strcmp (name, ".") == 0) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid self-referential filename '.'"); - return FALSE; + return glnx_throw (error, "Invalid self-referential filename '.'"); } if (strcmp (name, "..") == 0) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid path uplink filename '..'"); - return FALSE; + return glnx_throw (error, "Invalid path uplink filename '..'"); } if (strchr (name, '/') != NULL) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid / in filename %s", name); - return FALSE; + return glnx_throw (error, "Invalid / in filename %s", name); } return TRUE; } diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 4b0d8821..c14cbec5 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -147,9 +147,7 @@ handle_statoverride_line (const char *line, spc = strchr (line, ' '); if (spc == NULL) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Malformed statoverride file (no space found)"); - return FALSE; + return glnx_throw (error, "Malformed statoverride file (no space found)"); } mode_add = (guint32)(gint32)g_ascii_strtod (line, NULL); diff --git a/src/ostree/ot-builtin-config.c b/src/ostree/ot-builtin-config.c index 0d8f7b77..a9a5f52a 100644 --- a/src/ostree/ot-builtin-config.c +++ b/src/ostree/ot-builtin-config.c @@ -41,9 +41,8 @@ split_key_string (const char *k, if (!dot) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Key must be of the form \"sectionname.keyname\""); - return FALSE; + return glnx_throw (error, + "Key must be of the form \"sectionname.keyname\""); } *out_section = g_strndup (k, dot - k); From 9a3555a74b29bf980545b8eb45e0ea2070cf33f7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 26 May 2017 17:53:49 -0400 Subject: [PATCH 26/86] cmd: Use autoptr for GKeyFile Prep for code style conversion. Closes: #891 Approved by: jlebon --- src/ostree/ot-admin-builtin-deploy.c | 4 +--- src/ostree/ot-admin-builtin-switch.c | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index e59bec8e..709c635e 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -56,7 +56,7 @@ ot_admin_builtin_deploy (int argc, char **argv, GCancellable *cancellable, GErro const char *refspec; g_autoptr(GOptionContext) context = NULL; glnx_unref_object OstreeSysroot *sysroot = NULL; - GKeyFile *origin = NULL; + g_autoptr(GKeyFile) origin = NULL; glnx_unref_object OstreeRepo *repo = NULL; glnx_unref_object OstreeDeployment *new_deployment = NULL; glnx_unref_object OstreeDeployment *merge_deployment = NULL; @@ -171,7 +171,5 @@ ot_admin_builtin_deploy (int argc, char **argv, GCancellable *cancellable, GErro ret = TRUE; out: - if (origin) - g_key_file_unref (origin); return ret; } diff --git a/src/ostree/ot-admin-builtin-switch.c b/src/ostree/ot-admin-builtin-switch.c index d72aeebd..9df77f05 100644 --- a/src/ostree/ot-admin-builtin-switch.c +++ b/src/ostree/ot-admin-builtin-switch.c @@ -58,7 +58,7 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro glnx_unref_object OstreeAsyncProgress *progress = NULL; gboolean changed; GKeyFile *old_origin; - GKeyFile *new_origin = NULL; + g_autoptr(GKeyFile) new_origin = NULL; context = g_option_context_new ("REF - Construct new tree from REF and deploy it"); @@ -166,7 +166,5 @@ ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GErro ret = TRUE; out: - if (new_origin) - g_key_file_unref (new_origin); return ret; } From 241470460922c71e69beb3e0ac09c3840f33e200 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 30 May 2017 13:04:18 -0400 Subject: [PATCH 27/86] lib/util: Some style conversion I saw a few instances of `glnx_set_error_from_errno() + return FALSE`, and fixed them and did a bit of style conversion. Closes: #895 Approved by: jlebon --- src/libotutil/ot-fs-utils.c | 59 ++++++++---------------------------- src/libotutil/ot-gio-utils.c | 5 +-- 2 files changed, 13 insertions(+), 51 deletions(-) diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index fe185e66..8ed88984 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -97,24 +97,15 @@ ot_readlinkat_gfile_info (int dfd, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; char targetbuf[PATH_MAX+1]; ssize_t len; - do - len = readlinkat (dfd, path, targetbuf, sizeof (targetbuf) - 1); - while (G_UNLIKELY (len == -1 && errno == EINTR)); - if (len == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + if (TEMP_FAILURE_RETRY (len = readlinkat (dfd, path, targetbuf, sizeof (targetbuf) - 1)) < 0) + return glnx_throw_errno_prefix (error, "readlinkat"); targetbuf[len] = '\0'; g_file_info_set_symlink_target (target_info, targetbuf); - ret = TRUE; - out: - return ret; + return TRUE; } @@ -140,26 +131,17 @@ ot_openat_read_stream (int dfd, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; int fd = -1; int flags = O_RDONLY | O_NOCTTY | O_CLOEXEC; if (!follow) flags |= O_NOFOLLOW; - do - fd = openat (dfd, path, flags, 0); - while (G_UNLIKELY (fd == -1 && errno == EINTR)); - if (fd == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + if (TEMP_FAILURE_RETRY (fd = openat (dfd, path, flags, 0)) < 0) + return glnx_throw_errno_prefix (error, "openat(%s)", path); *out_istream = g_unix_input_stream_new (fd, TRUE); - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -170,10 +152,7 @@ ot_ensure_unlinked_at (int dfd, if (unlinkat (dfd, path, 0) != 0) { if (G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "unlink(%s)", path); } return TRUE; } @@ -189,10 +168,7 @@ ot_query_exists_at (int dfd, const char *path, if (fstatat (dfd, path, &stbuf, AT_SYMLINK_NOFOLLOW) < 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "fstatat(%s)", path); ret_exists = FALSE; } else @@ -208,23 +184,15 @@ ot_openat_ignore_enoent (int dfd, int *out_fd, GError **error) { - gboolean ret = FALSE; - int target_fd = -1; - - target_fd = openat (dfd, path, O_CLOEXEC | O_RDONLY); + int target_fd = openat (dfd, path, O_CLOEXEC | O_RDONLY); if (target_fd < 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "openat(%s)", path); } - ret = TRUE; *out_fd = target_fd; - out: - return ret; + return TRUE; } /* Like glnx_dirfd_iterator_init_at(), but if %ENOENT, then set @@ -261,10 +229,7 @@ ot_file_mapat_bytes (int dfd, g_autoptr(GMappedFile) mfile = NULL; if (fd < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_null_throw_errno_prefix (error, "openat(%s)", path); mfile = g_mapped_file_new_from_fd (fd, FALSE, error); if (!mfile) diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c index 5d12d430..a4b61842 100644 --- a/src/libotutil/ot-gio-utils.c +++ b/src/libotutil/ot-gio-utils.c @@ -100,10 +100,7 @@ ot_gfile_ensure_unlinked (GFile *path, if (unlink (gs_file_get_path_cached (path)) != 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "unlink(%s)", gs_file_get_path_cached (path)); } return TRUE; } From 1eff3e83436b6129c0dc350dbbda52ba330e3834 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 30 May 2017 14:07:13 -0400 Subject: [PATCH 28/86] Add a notion of "physical" sysroot, use for remote writing Using `${sysroot}` to mean the physical storage root: We don't want to write to `${sysroot}/etc/ostree/remotes.d`, since nothing will read it, and really `${sysroot}` should just have `/ostree` (ideally). Today the Anaconda rpmostree code ends up writing there. Fix this by adding a notion of "physical" sysroot. We determine whether the path is physical by checking for `/sysroot`, which exists in deployment roots (and there shouldn't be a `${sysroot}/sysroot`). In order to unit test this, I added a `--sysroot` argument to `remote add`. However, doing this better would require reworking the command line parsing for the `remote` argument to support specifying `--repo` or `--sysroot`, and I didn't quite want to do that yet in this patch. Closes: https://github.com/ostreedev/ostree/issues/892 Closes: #896 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 39 +++++++++++++------------- src/libostree/ostree-sysroot-private.h | 3 +- src/libostree/ostree-sysroot.c | 22 +++++++++++++++ src/ostree/ot-remote-builtin-add.c | 17 +++++++++++ tests/admin-test.sh | 16 +++++++++++ tests/test-admin-deploy-grub2.sh | 2 +- tests/test-admin-deploy-syslinux.sh | 2 +- tests/test-admin-deploy-uboot.sh | 2 +- 9 files changed, 81 insertions(+), 23 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 015a9a8b..6a7e9ee8 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -93,6 +93,7 @@ struct OstreeRepo { int objects_dir_fd; int uncompressed_objects_dir_fd; GFile *sysroot_dir; + GWeakRef sysroot; /* Weak to avoid a circular ref; see also `is_system` */ char *remotes_config_dir; GHashTable *txn_refs; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index cbbaec9b..4ad112df 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -32,6 +32,7 @@ #include #include "ostree-core-private.h" +#include "ostree-sysroot-private.h" #include "ostree-remote-private.h" #include "ostree-repo-private.h" #include "ostree-repo-file.h" @@ -447,6 +448,7 @@ ostree_repo_finalize (GObject *object) if (self->uncompressed_objects_dir_fd != -1) (void) close (self->uncompressed_objects_dir_fd); g_clear_object (&self->sysroot_dir); + g_weak_ref_clear (&self->sysroot); g_free (self->remotes_config_dir); if (self->loose_object_devino_hash) @@ -525,10 +527,6 @@ ostree_repo_constructed (GObject *object) g_assert (self->repodir != NULL); - /* Ensure the "sysroot-path" property is set. */ - if (self->sysroot_dir == NULL) - self->sysroot_dir = g_object_ref (_ostree_get_default_sysroot_path ()); - G_OBJECT_CLASS (ostree_repo_parent_class)->constructed (object); } @@ -706,8 +704,6 @@ ostree_repo_new_default (void) gboolean ostree_repo_is_system (OstreeRepo *repo) { - g_autoptr(GFile) default_repo_path = NULL; - g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); /* If we were created via ostree_sysroot_get_repo(), we know the answer is yes @@ -716,8 +712,11 @@ ostree_repo_is_system (OstreeRepo *repo) if (repo->is_system) return TRUE; - default_repo_path = get_default_repo_path (repo->sysroot_dir); + /* No sysroot_dir set? Not a system repo then. */ + if (!repo->sysroot_dir) + return FALSE; + g_autoptr(GFile) default_repo_path = get_default_repo_path (repo->sysroot_dir); return g_file_equal (repo->repodir, default_repo_path); } @@ -894,23 +893,25 @@ impl_repo_remote_add (OstreeRepo *self, remote = ostree_remote_new (name); - /* The OstreeRepo maintains its own internal system root path, - * so we need to not only check if a "sysroot" argument was given - * but also whether it's actually different from OstreeRepo's. - * - * XXX Having API regret about the "sysroot" argument now. + /* If a sysroot was provided, use it. Otherwise, see if this repo has a ref to + * a sysroot (and it's physical). */ - gboolean different_sysroot = FALSE; - if (sysroot != NULL) - different_sysroot = !g_file_equal (sysroot, self->sysroot_dir); + g_autoptr(OstreeSysroot) sysroot_ref = NULL; + if (sysroot == NULL) + { + sysroot_ref = (OstreeSysroot*)g_weak_ref_get (&self->sysroot); + /* Only write to /etc/ostree/remotes.d if we are pointed at a deployment */ + if (sysroot_ref != NULL && !sysroot_ref->is_physical) + sysroot = ostree_sysroot_get_path (sysroot_ref); + } + /* For backwards compat, also fall back to the sysroot-path variable */ + if (sysroot == NULL && sysroot_ref == NULL) + sysroot = self->sysroot_dir; - if (different_sysroot || ostree_repo_is_system (self)) + if (sysroot != NULL) { g_autoptr(GError) local_error = NULL; - if (sysroot == NULL) - sysroot = self->sysroot_dir; - g_autoptr(GFile) etc_ostree_remotes_d = g_file_resolve_relative_path (sysroot, SYSCONF_REMOTES); if (!g_file_make_directory_with_parents (etc_ostree_remotes_d, cancellable, &local_error)) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 14ee5cad..82abc8e7 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -48,7 +48,8 @@ struct OstreeSysroot { GLnxLockFile lock; gboolean loaded; - + + gboolean is_physical; /* TRUE if we're pointed at physical storage root and not a deployment */ GPtrArray *deployments; int bootversion; int subbootversion; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 86aa8ce3..b8f494bf 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -135,6 +135,8 @@ ostree_sysroot_constructed (GObject *object) repo_path = g_file_resolve_relative_path (self->path, "ostree/repo"); self->repo = ostree_repo_new_for_sysroot_path (repo_path, self->path); self->repo->is_system = TRUE; + /* Hold a weak ref for the remote-add handling */ + g_weak_ref_init (&self->repo->sysroot, object); G_OBJECT_CLASS (ostree_sysroot_parent_class)->constructed (object); } @@ -813,6 +815,26 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, cancellable, error)) return FALSE; + /* Determine whether we're "physical" or not, the first time we initialize */ + if (!self->loaded) + { + /* If we have a booted deployment, the sysroot is / and we're definitely + * not physical. + */ + if (self->booted_deployment) + self->is_physical = FALSE; /* (the default, but explicit for clarity) */ + /* Otherwise - check for /sysroot which should only exist in a deployment, + * not in ${sysroot} (a metavariable for the real physical root). + */ + else if (fstatat (self->sysroot_fd, "sysroot", &stbuf, 0) < 0) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "fstatat"); + self->is_physical = TRUE; + } + /* Otherwise, the default is FALSE */ + } + self->bootversion = bootversion; self->subbootversion = subbootversion; self->deployments = deployments; diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 3e3aeda9..9639b4a5 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -31,6 +31,7 @@ static gboolean opt_no_gpg_verify; static gboolean opt_if_not_exists; static char *opt_gpg_import; static char *opt_contenturl; +static char *opt_sysroot; static GOptionEntry option_entries[] = { { "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, @@ -38,6 +39,7 @@ static GOptionEntry option_entries[] = { { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, { "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" }, + { "sysroot", 0, 0, G_OPTION_ARG_FILENAME, &opt_sysroot, "Use sysroot at PATH (overrides --repo)", "PATH" }, { NULL } }; @@ -51,6 +53,7 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError char **iter; g_autoptr(GVariantBuilder) optbuilder = NULL; g_autoptr(GVariant) options = NULL; + g_autoptr(OstreeSysroot) sysroot = NULL; gboolean ret = FALSE; context = g_option_context_new ("NAME [metalink=|mirrorlist=]URL [BRANCH...] - Add a remote repository"); @@ -59,6 +62,20 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) goto out; + /* As a special case, we can take a --sysroot argument. Currently we also + * require --repo because fixing that needs more cmdline rework. + */ + if (opt_sysroot) + { + g_clear_object (&repo); + g_autoptr(GFile) sysroot_path = g_file_new_for_path (opt_sysroot); + sysroot = ostree_sysroot_new (sysroot_path); + if (!ostree_sysroot_load (sysroot, cancellable, error)) + goto out; + if (!ostree_sysroot_get_repo (sysroot, &repo, cancellable, error)) + goto out; + } + if (argc < 3) { ot_util_usage_error (context, "NAME and URL must be specified", error); diff --git a/tests/admin-test.sh b/tests/admin-test.sh index cc06fe6f..7182e60b 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -232,3 +232,19 @@ curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buil assert_streq ${curr_rev} ${head_rev} echo "ok upgrade with and without override-commit" + +deployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir) +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo --sysroot=sysroot remote add --set=gpg-verify=false remote-test-physical file://$(pwd)/testos-repo +assert_not_has_file ${deployment}/etc/ostree/remotes.d/remote-test-physical.conf testos-repo +assert_file_has_content sysroot/ostree/repo/config remote-test-physical +echo "ok remote add physical sysroot" + +# Now a hack...symlink ${deployment}/sysroot to the sysroot in lieu of a bind +# mount which we can't do in unit tests. +ln -sr sysroot ${deployment}/sysroot +ln -s sysroot/ostree ${deployment}/ostree +ls -al ${deployment} +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo --sysroot=${deployment} remote add --set=gpg-verify=false remote-test-nonphysical file://$(pwd)/testos-repo +assert_not_file_has_content sysroot/ostree/repo/config remote-test-nonphysical +assert_file_has_content ${deployment}/etc/ostree/remotes.d/remote-test-nonphysical.conf testos-repo +echo "ok remote add nonphysical sysroot" diff --git a/tests/test-admin-deploy-grub2.sh b/tests/test-admin-deploy-grub2.sh index 2b90c286..d7c1c6db 100755 --- a/tests/test-admin-deploy-grub2.sh +++ b/tests/test-admin-deploy-grub2.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..16" +echo "1..18" . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-syslinux.sh b/tests/test-admin-deploy-syslinux.sh index 70b3b4d3..797836f0 100755 --- a/tests/test-admin-deploy-syslinux.sh +++ b/tests/test-admin-deploy-syslinux.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..16" +echo "1..18" . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-uboot.sh b/tests/test-admin-deploy-uboot.sh index d4c3a0db..d9104f8c 100755 --- a/tests/test-admin-deploy-uboot.sh +++ b/tests/test-admin-deploy-uboot.sh @@ -20,7 +20,7 @@ set -euo pipefail -echo "1..17" +echo "1..19" . $(dirname $0)/libtest.sh From 22b1234f52895f31b59f6b8b31659b7ee7879eed Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 May 2017 14:49:17 -0400 Subject: [PATCH 29/86] repo/commit: Dedup metadata writing API implementations First, the streaming metadata API is pretty dumb, since metadata should be small. Really we should have supported a `GBytes` version. Currently, this API *is* used when we do local pulls, so this commit has test coverage. However, I plan to change the object import to avoid using this. But that's fine, since I can't think of why someone would use this API. Next, the only difference between `ostree_repo_write_metadata()` and `ostree_repo_write_metadata_trusted()` is whether or not we pass an output checksum; so just dedup the implementations. Also while I'm here break out the input length validation and do it early in the streaming case. Closes: #881 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 65 +++++++++++++++++++----------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index ce45a911..c788f7e6 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1481,6 +1481,25 @@ ostree_repo_abort_transaction (OstreeRepo *self, return TRUE; } +/* These limits were introduced since in some cases we may be processing + * malicious metadata, and we want to make disk space exhaustion attacks harder. + */ +static gboolean +metadata_size_valid (OstreeObjectType objtype, + gsize len, + GError **error) +{ + if (G_UNLIKELY (len > OSTREE_MAX_METADATA_SIZE)) + { + g_autofree char *input_bytes = g_format_size (len); + g_autofree char *max_bytes = g_format_size (OSTREE_MAX_METADATA_SIZE); + return glnx_throw (error, "Metadata object of type '%s' is %s; maximum metadata size is %s", + ostree_object_type_to_string (objtype), input_bytes, max_bytes); + } + + return TRUE; +} + /** * ostree_repo_write_metadata: * @self: Repo @@ -1506,20 +1525,11 @@ ostree_repo_write_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GVariant) normalized = NULL; - - normalized = g_variant_get_normal_form (object); - - if (G_UNLIKELY (g_variant_get_size (normalized) > OSTREE_MAX_METADATA_SIZE)) - { - g_autofree char *input_bytes = g_format_size (g_variant_get_size (normalized)); - g_autofree char *max_bytes = g_format_size (OSTREE_MAX_METADATA_SIZE); - return glnx_throw (error, "Metadata object of type '%s' is %s; maximum metadata size is %s", - ostree_object_type_to_string (objtype), input_bytes, max_bytes); - } + g_autoptr(GVariant) normalized = g_variant_get_normal_form (object); + if (!metadata_size_valid (objtype, g_variant_get_size (normalized), error)) + return FALSE; g_autoptr(GInputStream) input = ot_variant_read (normalized); - if (!write_object (self, objtype, expected_checksum, input, g_variant_get_size (normalized), out_csum, @@ -1551,8 +1561,22 @@ ostree_repo_write_metadata_stream_trusted (OstreeRepo *self, GCancellable *cancellable, GError **error) { - return write_object (self, objtype, checksum, object_input, length, NULL, - cancellable, error); + if (length > 0 && !metadata_size_valid (objtype, length, error)) + return FALSE; + + /* This is all pretty ridiculous, but we're keeping this API for backwards + * compatibility, it doesn't really need to be fast. + */ + g_autoptr(GMemoryOutputStream) tmpbuf = (GMemoryOutputStream*)g_memory_output_stream_new_resizable (); + if (g_output_stream_splice ((GOutputStream*)tmpbuf, object_input, + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, cancellable, error) < 0) + return FALSE; + g_autoptr(GBytes) tmpb = g_memory_output_stream_steal_as_bytes (tmpbuf); + + g_autoptr(GVariant) tmpv = g_variant_new_from_bytes (ostree_metadata_variant_type (objtype), + tmpb, TRUE); + return ostree_repo_write_metadata_trusted (self, objtype, checksum, tmpv, + cancellable, error); } /** @@ -1575,16 +1599,9 @@ ostree_repo_write_metadata_trusted (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GInputStream) input = NULL; - g_autoptr(GVariant) normalized = NULL; - - normalized = g_variant_get_normal_form (variant); - input = ot_variant_read (normalized); - - return write_object (self, type, checksum, - input, g_variant_get_size (normalized), - NULL, - cancellable, error); + return ostree_repo_write_metadata (self, type, + checksum, variant, NULL, + cancellable, error); } typedef struct { From d2a92df1552c979a562ce6c7a69a9fe0da08e55c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 May 2017 14:58:03 -0400 Subject: [PATCH 30/86] repo/commit: Dedup content writing API implementation Similar to metadata, for `write_content_trusted()` we can just call `_write_content()` with a `NULL` output checksum. Closes: #881 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index c788f7e6..a6d5f430 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1748,9 +1748,8 @@ ostree_repo_write_content_trusted (OstreeRepo *self, GCancellable *cancellable, GError **error) { - return write_object (self, OSTREE_OBJECT_TYPE_FILE, checksum, - object_input, length, NULL, - cancellable, error); + return ostree_repo_write_content (self, checksum, object_input, length, + NULL, cancellable, error); } /** From 6ba4dac6f21f5e613022e3bcce5635668735a4fa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 May 2017 15:04:50 -0400 Subject: [PATCH 31/86] repo/commit: In the expected checksum case, check existence early If we have an expected checksum, call `fstatat(repo_dfd, checksum)` early on before we do much else. This actually duplicates code, but future work here is going to split up the metadata/content commit paths, so they'll need to diverge anyways. Closes: #881 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 49 +++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index a6d5f430..fde25db5 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -625,20 +625,6 @@ write_object (OstreeRepo *self, if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - if (expected_checksum) - { - if (!_ostree_repo_has_loose_object (self, expected_checksum, objtype, &have_obj, - cancellable, error)) - goto out; - if (have_obj) - { - if (out_csum) - *out_csum = ostree_checksum_to_bytes (expected_checksum); - ret = TRUE; - goto out; - } - } - repo_mode = ostree_repo_get_mode (self); if (out_csum) @@ -1525,6 +1511,23 @@ ostree_repo_write_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error) { + /* First, if we have an expected checksum, see if we already have this + * object. This mirrors the same logic in ostree_repo_write_content(). + */ + if (expected_checksum) + { + gboolean have_obj; + if (!_ostree_repo_has_loose_object (self, expected_checksum, objtype, &have_obj, + cancellable, error)) + return FALSE; + if (have_obj) + { + if (out_csum) + *out_csum = ostree_checksum_to_bytes (expected_checksum); + return TRUE; + } + } + g_autoptr(GVariant) normalized = g_variant_get_normal_form (object); if (!metadata_size_valid (objtype, g_variant_get_size (normalized), error)) return FALSE; @@ -1775,6 +1778,24 @@ ostree_repo_write_content (OstreeRepo *self, GCancellable *cancellable, GError **error) { + /* First, if we have an expected checksum, see if we already have this + * object. This mirrors the same logic in ostree_repo_write_metadata(). + */ + if (expected_checksum) + { + gboolean have_obj; + if (!_ostree_repo_has_loose_object (self, expected_checksum, + OSTREE_OBJECT_TYPE_FILE, &have_obj, + cancellable, error)) + return FALSE; + if (have_obj) + { + if (out_csum) + *out_csum = ostree_checksum_to_bytes (expected_checksum); + return TRUE; + } + } + return write_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum, object_input, length, out_csum, cancellable, error); From ec1964dd4416e255d41d590e08bc247484f1f304 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 May 2017 15:13:08 -0400 Subject: [PATCH 32/86] repo/commit: Don't renormalize trusted metadata As the comment in the code says; in the expected checksum case, the caller really has to have a normal form already. Closes: #881 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index fde25db5..226ddcda 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1511,6 +1511,7 @@ ostree_repo_write_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error) { + g_autoptr(GVariant) normalized = NULL; /* First, if we have an expected checksum, see if we already have this * object. This mirrors the same logic in ostree_repo_write_content(). */ @@ -1526,9 +1527,17 @@ ostree_repo_write_metadata (OstreeRepo *self, *out_csum = ostree_checksum_to_bytes (expected_checksum); return TRUE; } + /* If the caller is giving us an expected checksum, the object really has + * to be normalized already. Otherwise, how would they know the checksum? + * There's no sense in redoing it. + */ + normalized = g_variant_ref (object); + } + else + { + normalized = g_variant_get_normal_form (object); } - g_autoptr(GVariant) normalized = g_variant_get_normal_form (object); if (!metadata_size_valid (objtype, g_variant_get_size (normalized), error)) return FALSE; From f4f13307896b350676e2b32751fb5e33aa0fb3a9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 23 May 2017 16:18:31 -0400 Subject: [PATCH 33/86] repo/commit: Split up metadata/content commit paths There was a lot of conditionals inside `write_object()` differentating between metadata/content, and then for content, on the different repo types. Further, in the metadata path since the logic is simpler, can present a non-streaming API, and further use `OtTmpfile`, etc. Splitting them up helps drop a lot of conditionals. We introduce a small `CleanupUnlinkat` that allows us to fully convert to the new code style in both functions. This itself is still prep for fully switching to `GLnxTmpfile`. Closes: #881 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 562 ++++++++++++++++------------- 1 file changed, 312 insertions(+), 250 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 226ddcda..870a06c7 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -591,292 +591,356 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, return TRUE; } -static gboolean -write_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *expected_checksum, - GInputStream *input, - guint64 file_object_length, - guchar **out_csum, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - const char *actual_checksum = NULL; - g_autofree char *actual_checksum_owned = NULL; - gboolean do_commit; - OstreeRepoMode repo_mode; - g_autofree char *temp_filename = NULL; - g_autofree guchar *ret_csum = NULL; - glnx_unref_object OtChecksumInstream *checksum_input = NULL; - g_autoptr(GInputStream) file_input = NULL; - g_autoptr(GFileInfo) file_info = NULL; - g_autoptr(GVariant) xattrs = NULL; - gboolean have_obj; - gboolean temp_file_is_regular; - gboolean temp_file_is_symlink; - glnx_fd_close int temp_fd = -1; - gboolean object_is_symlink = FALSE; - gssize unpacked_size = 0; - gboolean indexable = FALSE; +/* A little helper to call unlinkat() as a cleanup + * function. Mostly only necessary to handle + * deletion of temporary symlinks. + */ +typedef struct { + int dfd; + const char *path; +} CleanupUnlinkat; +static void +cleanup_unlinkat (CleanupUnlinkat *cleanup) +{ + if (cleanup->path) + (void) unlinkat (cleanup->dfd, cleanup->path, 0); +} +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(CleanupUnlinkat, cleanup_unlinkat); + +/* Write a content object. */ +static gboolean +write_content_object (OstreeRepo *self, + const char *expected_checksum, + GInputStream *input, + guint64 file_object_length, + guchar **out_csum, + GCancellable *cancellable, + GError **error) +{ g_return_val_if_fail (expected_checksum || out_csum, FALSE); if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - repo_mode = ostree_repo_get_mode (self); + OstreeRepoMode repo_mode = ostree_repo_get_mode (self); + glnx_unref_object OtChecksumInstream *checksum_input = NULL; if (out_csum) checksum_input = ot_checksum_instream_new (input, G_CHECKSUM_SHA256); - if (objtype == OSTREE_OBJECT_TYPE_FILE) + g_autoptr(GInputStream) file_input = NULL; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GFileInfo) file_info = NULL; + if (!ostree_content_stream_parse (FALSE, checksum_input ? (GInputStream*)checksum_input : input, + file_object_length, FALSE, + &file_input, &file_info, &xattrs, + cancellable, error)) + return FALSE; + + gboolean temp_file_is_regular = g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR; + gboolean temp_file_is_symlink = g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK; + gboolean object_is_symlink = temp_file_is_symlink; + + if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_is_symlink) { - if (!ostree_content_stream_parse (FALSE, checksum_input ? (GInputStream*)checksum_input : input, - file_object_length, FALSE, - &file_input, &file_info, &xattrs, - cancellable, error)) - goto out; + const char *target_str = g_file_info_get_symlink_target (file_info); + g_autoptr(GBytes) target = g_bytes_new (target_str, strlen (target_str) + 1); - temp_file_is_regular = g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR; - temp_file_is_symlink = object_is_symlink = - g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK; + /* For bare-user we can't store symlinks as symlinks, as symlinks don't + support user xattrs to store the ownership. So, instead store them + as regular files */ + temp_file_is_regular = TRUE; + temp_file_is_symlink = FALSE; - if (repo_mode == OSTREE_REPO_MODE_BARE_USER && object_is_symlink) - { - const char *target_str = g_file_info_get_symlink_target (file_info); - g_autoptr(GBytes) target = g_bytes_new (target_str, strlen (target_str) + 1); - - /* For bare-user we can't store symlinks as symlinks, as symlinks don't - support user xattrs to store the ownership. So, instead store them - as regular files */ - temp_file_is_regular = TRUE; - temp_file_is_symlink = FALSE; - if (file_input != NULL) - g_object_unref (file_input); - - /* Include the terminating zero so we can e.g. mmap this file */ - file_input = g_memory_input_stream_new_from_bytes (target); - } - - if (!(temp_file_is_regular || temp_file_is_symlink)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unsupported file type %u", g_file_info_get_file_type (file_info)); - goto out; - } - - /* For regular files, we create them with default mode, and only - * later apply any xattrs and setuid bits. The rationale here - * is that an attacker on the network with the ability to MITM - * could potentially cause the system to make a temporary setuid - * binary with trailing garbage, creating a window on the local - * system where a malicious setuid binary exists. - */ - if ((_ostree_repo_mode_is_bare (repo_mode)) && temp_file_is_regular) - { - guint64 size = g_file_info_get_size (file_info); - - if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, - &temp_fd, &temp_filename, - cancellable, error)) - goto out; - } - else if (_ostree_repo_mode_is_bare (repo_mode) && temp_file_is_symlink) - { - /* Note: This will not be hit for bare-user mode because its converted to a - regular file and take the branch above */ - if (!_ostree_make_temporary_symlink_at (self->tmp_dir_fd, - g_file_info_get_symlink_target (file_info), - &temp_filename, - cancellable, error)) - goto out; - } - else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) - { - g_autoptr(GVariant) file_meta = NULL; - g_autoptr(GConverter) zlib_compressor = NULL; - g_autoptr(GOutputStream) compressed_out_stream = NULL; - g_autoptr(GOutputStream) temp_out = NULL; - - if (self->generate_sizes) - indexable = TRUE; - - if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, - &temp_fd, &temp_filename, - error)) - goto out; - temp_file_is_regular = TRUE; - temp_out = g_unix_output_stream_new (temp_fd, FALSE); - - file_meta = _ostree_zlib_file_header_new (file_info, xattrs); - - if (!_ostree_write_variant_with_size (temp_out, file_meta, 0, NULL, NULL, - cancellable, error)) - goto out; - - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - { - zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level); - compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor); - /* Don't close the base; we'll do that later */ - g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE); - - unpacked_size = g_output_stream_splice (compressed_out_stream, file_input, - 0, cancellable, error); - if (unpacked_size < 0) - goto out; - } - - if (!g_output_stream_flush (temp_out, cancellable, error)) - goto out; - - if (fchmod (temp_fd, 0644) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } - } - else - g_assert_not_reached (); + if (file_input != NULL) + g_object_unref (file_input); + /* Include the terminating zero so we can e.g. mmap this file */ + file_input = g_memory_input_stream_new_from_bytes (target); } - else + + if (!(temp_file_is_regular || temp_file_is_symlink)) + return glnx_throw (error, "Unsupported file type %u", g_file_info_get_file_type (file_info)); + + /* For regular files, we create them with default mode, and only + * later apply any xattrs and setuid bits. The rationale here + * is that an attacker on the network with the ability to MITM + * could potentially cause the system to make a temporary setuid + * binary with trailing garbage, creating a window on the local + * system where a malicious setuid binary exists. + */ + /* These variables are almost equivalent to OtTmpfile, except + * temp_filename might also be a symlink. Hence the CleanupUnlinkat + * which handles that case. + */ + g_auto(CleanupUnlinkat) tmp_unlinker = { self->tmp_dir_fd, NULL }; + glnx_fd_close int temp_fd = -1; + g_autofree char *temp_filename = NULL; + gssize unpacked_size = 0; + gboolean indexable = FALSE; + if ((_ostree_repo_mode_is_bare (repo_mode)) && temp_file_is_regular) { - if (!create_regular_tmpfile_linkable_with_content (self, file_object_length, - checksum_input ? (GInputStream*)checksum_input : input, + guint64 size = g_file_info_get_size (file_info); + + if (!create_regular_tmpfile_linkable_with_content (self, size, file_input, &temp_fd, &temp_filename, cancellable, error)) - goto out; + return FALSE; + tmp_unlinker.path = temp_filename; + } + else if (_ostree_repo_mode_is_bare (repo_mode) && temp_file_is_symlink) + { + /* Note: This will not be hit for bare-user mode because its converted to a + regular file and take the branch above */ + if (!_ostree_make_temporary_symlink_at (self->tmp_dir_fd, + g_file_info_get_symlink_target (file_info), + &temp_filename, + cancellable, error)) + return FALSE; + tmp_unlinker.path = temp_filename; + } + else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) + { + g_autoptr(GVariant) file_meta = NULL; + g_autoptr(GConverter) zlib_compressor = NULL; + g_autoptr(GOutputStream) compressed_out_stream = NULL; + g_autoptr(GOutputStream) temp_out = NULL; + + if (self->generate_sizes) + indexable = TRUE; + + if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, + &temp_fd, &temp_filename, + error)) + return FALSE; + tmp_unlinker.path = temp_filename; temp_file_is_regular = TRUE; + temp_out = g_unix_output_stream_new (temp_fd, FALSE); + + file_meta = _ostree_zlib_file_header_new (file_info, xattrs); + + if (!_ostree_write_variant_with_size (temp_out, file_meta, 0, NULL, NULL, + cancellable, error)) + return FALSE; + + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + { + zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level); + compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor); + /* Don't close the base; we'll do that later */ + g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE); + + unpacked_size = g_output_stream_splice (compressed_out_stream, file_input, + 0, cancellable, error); + if (unpacked_size < 0) + return FALSE; + } + + if (!g_output_stream_flush (temp_out, cancellable, error)) + return FALSE; + + if (fchmod (temp_fd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); } + const char *actual_checksum = NULL; + g_autofree char *actual_checksum_owned = NULL; if (!checksum_input) actual_checksum = expected_checksum; else { actual_checksum = actual_checksum_owned = ot_checksum_instream_get_string (checksum_input); if (expected_checksum && strcmp (actual_checksum, expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted %s object %s (actual checksum is %s)", - ostree_object_type_to_string (objtype), - expected_checksum, actual_checksum); - goto out; - } + return glnx_throw (error, "Corrupted %s object %s (actual checksum is %s)", + ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE), + expected_checksum, actual_checksum); } g_assert (actual_checksum != NULL); /* Pacify static analysis */ - + + /* See whether or not we have the object, now that we know the + * checksum. + */ + gboolean have_obj; + if (!_ostree_repo_has_loose_object (self, actual_checksum, OSTREE_OBJECT_TYPE_FILE, + &have_obj, cancellable, error)) + return FALSE; + /* If we already have it, just update the stats. */ + if (have_obj) + { + g_mutex_lock (&self->txn_stats_lock); + self->txn_stats.content_objects_total++; + g_mutex_unlock (&self->txn_stats_lock); + if (out_csum) + *out_csum = ostree_checksum_to_bytes (actual_checksum); + /* Note early return */ + return TRUE; + } + + const guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); + const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); + const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); + if (!commit_loose_object_trusted (self, actual_checksum, + OSTREE_OBJECT_TYPE_FILE, + temp_filename, + object_is_symlink, + uid, gid, mode, + xattrs, temp_fd, + cancellable, error)) + return FALSE; + /* Clear the unlinker path, it was consumed */ + tmp_unlinker.path = NULL; + + /* Update size metadata if configured */ if (indexable && temp_file_is_regular) { struct stat stbuf; - if (fstat (temp_fd, &stbuf) == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + if (!glnx_fstat (temp_fd, &stbuf, error)) + return FALSE; repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); } - if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, &have_obj, - cancellable, error)) - goto out; - - do_commit = !have_obj; - - if (do_commit) - { - guint32 uid, gid, mode; - - if (file_info) - { - uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); - gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); - mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); - } - else - uid = gid = mode = 0; - - if (!commit_loose_object_trusted (self, actual_checksum, objtype, - temp_filename, - object_is_symlink, - uid, gid, mode, - xattrs, temp_fd, - cancellable, error)) - goto out; - - - if (objtype == OSTREE_OBJECT_TYPE_COMMIT) - { - GError *local_error = NULL; - /* If we are writing a commit, be sure there is no tombstone for it. - We may have deleted the commit and now we are trying to pull it again. */ - if (!ostree_repo_delete_object (self, - OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT, - actual_checksum, - cancellable, - &local_error)) - { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - g_clear_error (&local_error); - else - { - g_propagate_error (error, local_error); - goto out; - } - } - } - - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - if (G_UNLIKELY (file_object_length > OSTREE_MAX_METADATA_WARN_SIZE)) - { - g_autofree char *metasize = g_format_size (file_object_length); - g_autofree char *warnsize = g_format_size (OSTREE_MAX_METADATA_WARN_SIZE); - g_autofree char *maxsize = g_format_size (OSTREE_MAX_METADATA_SIZE); - g_warning ("metadata object %s is %s, which is larger than the warning threshold of %s." \ - " The hard limit on metadata size is %s. Put large content in the tree itself, not in metadata.", - actual_checksum, - metasize, warnsize, maxsize); - } - } - - g_clear_pointer (&temp_filename, g_free); - } - + /* Update statistics */ g_mutex_lock (&self->txn_stats_lock); - if (do_commit) - { - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - self->txn_stats.metadata_objects_written++; - } - else - { - self->txn_stats.content_objects_written++; - self->txn_stats.content_bytes_written += file_object_length; - } - } - if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - self->txn_stats.metadata_objects_total++; - else - self->txn_stats.content_objects_total++; + self->txn_stats.content_objects_written++; + self->txn_stats.content_bytes_written += file_object_length; + self->txn_stats.content_objects_total++; g_mutex_unlock (&self->txn_stats_lock); - if (checksum_input) + if (out_csum) { g_assert (actual_checksum); - ret_csum = ostree_checksum_to_bytes (actual_checksum); + *out_csum = ostree_checksum_to_bytes (actual_checksum); } - ret = TRUE; - ot_transfer_out_value(out_csum, &ret_csum); - out: - if (temp_filename) - (void) unlinkat (self->tmp_dir_fd, temp_filename, 0); - return ret; + return TRUE; +} + +static gboolean +write_metadata_object (OstreeRepo *self, + OstreeObjectType objtype, + const char *expected_checksum, + GBytes *buf, + guchar **out_csum, + GCancellable *cancellable, + GError **error) +{ + g_return_val_if_fail (expected_checksum || out_csum, FALSE); + + if (g_cancellable_set_error_if_cancelled (cancellable, error)) + return FALSE; + + /* In the metadata case, we're not streaming, so we don't bother creating a + * tempfile until we compute the checksum. Some metadata like dirmeta is + * commonly duplicated, and computing the checksum is going to be cheaper than + * making a tempfile. + * + * However, tombstone commit types don't make sense to checksum, because for + * historical reasons we used ostree_repo_write_metadata_trusted() with the + * *original* sha256 to say what commit was being killed. + */ + const gboolean is_tombstone = (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT); + g_autofree char *actual_checksum = NULL; + if (is_tombstone) + { + actual_checksum = g_strdup (expected_checksum); + } + else + { + actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, buf); + gboolean have_obj; + if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, &have_obj, + cancellable, error)) + return FALSE; + /* If we already have the object, we just need to update the tried-to-commit + * stat for metadata and be done here. + */ + if (have_obj) + { + g_mutex_lock (&self->txn_stats_lock); + self->txn_stats.metadata_objects_total++; + g_mutex_unlock (&self->txn_stats_lock); + + if (out_csum) + *out_csum = ostree_checksum_to_bytes (actual_checksum); + /* Note early return */ + return TRUE; + } + + if (expected_checksum && strcmp (actual_checksum, expected_checksum) != 0) + return glnx_throw (error, "Corrupted %s object %s (actual checksum is %s)", + ostree_object_type_to_string (objtype), + expected_checksum, actual_checksum); + } + + /* Ok, checksum is known, let's get the data */ + gsize len; + const guint8 *bufp = g_bytes_get_data (buf, &len); + + /* Do the size warning here, to avoid warning for already extant metadata */ + if (G_UNLIKELY (len > OSTREE_MAX_METADATA_WARN_SIZE)) + { + g_autofree char *metasize = g_format_size (len); + g_autofree char *warnsize = g_format_size (OSTREE_MAX_METADATA_WARN_SIZE); + g_autofree char *maxsize = g_format_size (OSTREE_MAX_METADATA_SIZE); + g_warning ("metadata object %s is %s, which is larger than the warning threshold of %s." \ + " The hard limit on metadata size is %s. Put large content in the tree itself, not in metadata.", + actual_checksum, + metasize, warnsize, maxsize); + } + + /* Write the metadata to a temporary file */ + g_auto(OtTmpfile) tmpf = { 0, }; + if (!ot_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC, + &tmpf, error)) + return FALSE; + if (!ot_fallocate (tmpf.fd, len, error)) + return FALSE; + if (glnx_loop_write (tmpf.fd, bufp, len) < 0) + return glnx_throw_errno_prefix (error, "write()"); + if (fchmod (tmpf.fd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + + /* And commit it into place */ + if (!_ostree_repo_commit_loose_final (self, actual_checksum, objtype, + self->tmp_dir_fd, tmpf.fd, tmpf.path, + cancellable, error)) + return FALSE; + /* The temp path was consumed */ + g_clear_pointer (&tmpf.path, g_free); + + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + { + GError *local_error = NULL; + /* If we are writing a commit, be sure there is no tombstone for it. + We may have deleted the commit and now we are trying to pull it again. */ + if (!ostree_repo_delete_object (self, + OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT, + actual_checksum, + cancellable, + &local_error)) + { + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + g_clear_error (&local_error); + else + { + g_propagate_error (error, local_error); + return FALSE; + } + } + } + + /* Update the stats, note we both wrote one and add to total */ + g_mutex_lock (&self->txn_stats_lock); + self->txn_stats.metadata_objects_written++; + self->txn_stats.metadata_objects_total++; + g_mutex_unlock (&self->txn_stats_lock); + + if (out_csum) + *out_csum = ostree_checksum_to_bytes (actual_checksum); + return TRUE; } static gboolean @@ -1541,11 +1605,9 @@ ostree_repo_write_metadata (OstreeRepo *self, if (!metadata_size_valid (objtype, g_variant_get_size (normalized), error)) return FALSE; - g_autoptr(GInputStream) input = ot_variant_read (normalized); - if (!write_object (self, objtype, expected_checksum, - input, g_variant_get_size (normalized), - out_csum, - cancellable, error)) + g_autoptr(GBytes) vdata = g_variant_get_data_as_bytes (normalized); + if (!write_metadata_object (self, objtype, expected_checksum, + vdata, out_csum, cancellable, error)) return FALSE; return TRUE; @@ -1805,9 +1867,9 @@ ostree_repo_write_content (OstreeRepo *self, } } - return write_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum, - object_input, length, out_csum, - cancellable, error); + return write_content_object (self, expected_checksum, + object_input, length, out_csum, + cancellable, error); } typedef struct { From a094879f3a65ff0b9909dcb7f436257f034d29cd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 1 Jun 2017 15:55:15 -0400 Subject: [PATCH 34/86] lib/repo: Delete unused private prototypes The implementations were removed in: 6ffcb24d227eae5a479caf45adb8037eceb6ae33 I noticed this while looking at the commit code. Closes: #898 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 6a7e9ee8..67c9e44c 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -320,28 +320,6 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean -_ostree_repo_open_untrusted_content_bare (OstreeRepo *self, - const char *expected_checksum, - guint64 content_len, - OstreeRepoContentBareCommit *out_state, - GOutputStream **out_stream, - gboolean *out_have_object, - GCancellable *cancellable, - GError **error); - -gboolean -_ostree_repo_commit_untrusted_content_bare (OstreeRepo *self, - const char *expected_checksum, - OstreeRepoContentBareCommit *state, - guint32 uid, - guint32 gid, - guint32 mode, - GVariant *xattrs, - GCancellable *cancellable, - GError **error); - - gboolean _ostree_repo_read_bare_fd (OstreeRepo *self, const char *checksum, From 2fdbdd4b2f2b13b3b99cb2ef993e56445a5d1550 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 2 Jun 2017 14:50:29 +0200 Subject: [PATCH 35/86] lib/sysroot: Document the NO_CLEAN flag Closes: #900 Approved by: jlebon --- src/libostree/ostree-sysroot.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index b8f494bf..03e742f1 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1412,6 +1412,11 @@ ostree_sysroot_init_osname (OstreeSysroot *self, * If %OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NOT_DEFAULT is * specified, then instead of prepending, the new deployment will be * added right after the booted or merge deployment, instead of first. + * + * If %OSTREE_SYSROOT_SIMPLE_WRITE_DEPLOYMENT_FLAGS_NO_CLEAN is + * specified, then no cleanup will be performed after adding the + * deployment. Make sure to call ostree_sysroot_cleanup() sometime + * later, instead. */ gboolean ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, From cad42d960150db19bfaa518a06cea1febeb8fb58 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 2 Jun 2017 09:27:52 -0400 Subject: [PATCH 36/86] Revert "Add a notion of "physical" sysroot, use for remote writing" This reverts commit 1eff3e83436b6129c0dc350dbbda52ba330e3834. There are a few issues with it. It's not a critical thing for now, so let's ugly up the git history and revisit when we have time to debug it and add more tests. Besides the below issue, I noticed that the simple `ostree remote add` now writes to `/ostree/repo/config` because we *aren't* using the `--sysroot` argument. Closes: https://github.com/ostreedev/ostree/issues/901 Closes: #902 Approved by: mike-nguyen --- src/libostree/ostree-repo-private.h | 1 - src/libostree/ostree-repo.c | 39 +++++++++++++------------- src/libostree/ostree-sysroot-private.h | 3 +- src/libostree/ostree-sysroot.c | 22 --------------- src/ostree/ot-remote-builtin-add.c | 17 ----------- tests/admin-test.sh | 16 ----------- tests/test-admin-deploy-grub2.sh | 2 +- tests/test-admin-deploy-syslinux.sh | 2 +- tests/test-admin-deploy-uboot.sh | 2 +- 9 files changed, 23 insertions(+), 81 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 67c9e44c..55602940 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -93,7 +93,6 @@ struct OstreeRepo { int objects_dir_fd; int uncompressed_objects_dir_fd; GFile *sysroot_dir; - GWeakRef sysroot; /* Weak to avoid a circular ref; see also `is_system` */ char *remotes_config_dir; GHashTable *txn_refs; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 4ad112df..cbbaec9b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -32,7 +32,6 @@ #include #include "ostree-core-private.h" -#include "ostree-sysroot-private.h" #include "ostree-remote-private.h" #include "ostree-repo-private.h" #include "ostree-repo-file.h" @@ -448,7 +447,6 @@ ostree_repo_finalize (GObject *object) if (self->uncompressed_objects_dir_fd != -1) (void) close (self->uncompressed_objects_dir_fd); g_clear_object (&self->sysroot_dir); - g_weak_ref_clear (&self->sysroot); g_free (self->remotes_config_dir); if (self->loose_object_devino_hash) @@ -527,6 +525,10 @@ ostree_repo_constructed (GObject *object) g_assert (self->repodir != NULL); + /* Ensure the "sysroot-path" property is set. */ + if (self->sysroot_dir == NULL) + self->sysroot_dir = g_object_ref (_ostree_get_default_sysroot_path ()); + G_OBJECT_CLASS (ostree_repo_parent_class)->constructed (object); } @@ -704,6 +706,8 @@ ostree_repo_new_default (void) gboolean ostree_repo_is_system (OstreeRepo *repo) { + g_autoptr(GFile) default_repo_path = NULL; + g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); /* If we were created via ostree_sysroot_get_repo(), we know the answer is yes @@ -712,11 +716,8 @@ ostree_repo_is_system (OstreeRepo *repo) if (repo->is_system) return TRUE; - /* No sysroot_dir set? Not a system repo then. */ - if (!repo->sysroot_dir) - return FALSE; + default_repo_path = get_default_repo_path (repo->sysroot_dir); - g_autoptr(GFile) default_repo_path = get_default_repo_path (repo->sysroot_dir); return g_file_equal (repo->repodir, default_repo_path); } @@ -893,25 +894,23 @@ impl_repo_remote_add (OstreeRepo *self, remote = ostree_remote_new (name); - /* If a sysroot was provided, use it. Otherwise, see if this repo has a ref to - * a sysroot (and it's physical). + /* The OstreeRepo maintains its own internal system root path, + * so we need to not only check if a "sysroot" argument was given + * but also whether it's actually different from OstreeRepo's. + * + * XXX Having API regret about the "sysroot" argument now. */ - g_autoptr(OstreeSysroot) sysroot_ref = NULL; - if (sysroot == NULL) - { - sysroot_ref = (OstreeSysroot*)g_weak_ref_get (&self->sysroot); - /* Only write to /etc/ostree/remotes.d if we are pointed at a deployment */ - if (sysroot_ref != NULL && !sysroot_ref->is_physical) - sysroot = ostree_sysroot_get_path (sysroot_ref); - } - /* For backwards compat, also fall back to the sysroot-path variable */ - if (sysroot == NULL && sysroot_ref == NULL) - sysroot = self->sysroot_dir; - + gboolean different_sysroot = FALSE; if (sysroot != NULL) + different_sysroot = !g_file_equal (sysroot, self->sysroot_dir); + + if (different_sysroot || ostree_repo_is_system (self)) { g_autoptr(GError) local_error = NULL; + if (sysroot == NULL) + sysroot = self->sysroot_dir; + g_autoptr(GFile) etc_ostree_remotes_d = g_file_resolve_relative_path (sysroot, SYSCONF_REMOTES); if (!g_file_make_directory_with_parents (etc_ostree_remotes_d, cancellable, &local_error)) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 82abc8e7..14ee5cad 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -48,8 +48,7 @@ struct OstreeSysroot { GLnxLockFile lock; gboolean loaded; - - gboolean is_physical; /* TRUE if we're pointed at physical storage root and not a deployment */ + GPtrArray *deployments; int bootversion; int subbootversion; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 03e742f1..a3e9e75d 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -135,8 +135,6 @@ ostree_sysroot_constructed (GObject *object) repo_path = g_file_resolve_relative_path (self->path, "ostree/repo"); self->repo = ostree_repo_new_for_sysroot_path (repo_path, self->path); self->repo->is_system = TRUE; - /* Hold a weak ref for the remote-add handling */ - g_weak_ref_init (&self->repo->sysroot, object); G_OBJECT_CLASS (ostree_sysroot_parent_class)->constructed (object); } @@ -815,26 +813,6 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, cancellable, error)) return FALSE; - /* Determine whether we're "physical" or not, the first time we initialize */ - if (!self->loaded) - { - /* If we have a booted deployment, the sysroot is / and we're definitely - * not physical. - */ - if (self->booted_deployment) - self->is_physical = FALSE; /* (the default, but explicit for clarity) */ - /* Otherwise - check for /sysroot which should only exist in a deployment, - * not in ${sysroot} (a metavariable for the real physical root). - */ - else if (fstatat (self->sysroot_fd, "sysroot", &stbuf, 0) < 0) - { - if (errno != ENOENT) - return glnx_throw_errno_prefix (error, "fstatat"); - self->is_physical = TRUE; - } - /* Otherwise, the default is FALSE */ - } - self->bootversion = bootversion; self->subbootversion = subbootversion; self->deployments = deployments; diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 9639b4a5..3e3aeda9 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -31,7 +31,6 @@ static gboolean opt_no_gpg_verify; static gboolean opt_if_not_exists; static char *opt_gpg_import; static char *opt_contenturl; -static char *opt_sysroot; static GOptionEntry option_entries[] = { { "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, @@ -39,7 +38,6 @@ static GOptionEntry option_entries[] = { { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, { "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" }, - { "sysroot", 0, 0, G_OPTION_ARG_FILENAME, &opt_sysroot, "Use sysroot at PATH (overrides --repo)", "PATH" }, { NULL } }; @@ -53,7 +51,6 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError char **iter; g_autoptr(GVariantBuilder) optbuilder = NULL; g_autoptr(GVariant) options = NULL; - g_autoptr(OstreeSysroot) sysroot = NULL; gboolean ret = FALSE; context = g_option_context_new ("NAME [metalink=|mirrorlist=]URL [BRANCH...] - Add a remote repository"); @@ -62,20 +59,6 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) goto out; - /* As a special case, we can take a --sysroot argument. Currently we also - * require --repo because fixing that needs more cmdline rework. - */ - if (opt_sysroot) - { - g_clear_object (&repo); - g_autoptr(GFile) sysroot_path = g_file_new_for_path (opt_sysroot); - sysroot = ostree_sysroot_new (sysroot_path); - if (!ostree_sysroot_load (sysroot, cancellable, error)) - goto out; - if (!ostree_sysroot_get_repo (sysroot, &repo, cancellable, error)) - goto out; - } - if (argc < 3) { ot_util_usage_error (context, "NAME and URL must be specified", error); diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 7182e60b..cc06fe6f 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -232,19 +232,3 @@ curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buil assert_streq ${curr_rev} ${head_rev} echo "ok upgrade with and without override-commit" - -deployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir) -${CMD_PREFIX} ostree --repo=sysroot/ostree/repo --sysroot=sysroot remote add --set=gpg-verify=false remote-test-physical file://$(pwd)/testos-repo -assert_not_has_file ${deployment}/etc/ostree/remotes.d/remote-test-physical.conf testos-repo -assert_file_has_content sysroot/ostree/repo/config remote-test-physical -echo "ok remote add physical sysroot" - -# Now a hack...symlink ${deployment}/sysroot to the sysroot in lieu of a bind -# mount which we can't do in unit tests. -ln -sr sysroot ${deployment}/sysroot -ln -s sysroot/ostree ${deployment}/ostree -ls -al ${deployment} -${CMD_PREFIX} ostree --repo=sysroot/ostree/repo --sysroot=${deployment} remote add --set=gpg-verify=false remote-test-nonphysical file://$(pwd)/testos-repo -assert_not_file_has_content sysroot/ostree/repo/config remote-test-nonphysical -assert_file_has_content ${deployment}/etc/ostree/remotes.d/remote-test-nonphysical.conf testos-repo -echo "ok remote add nonphysical sysroot" diff --git a/tests/test-admin-deploy-grub2.sh b/tests/test-admin-deploy-grub2.sh index d7c1c6db..2b90c286 100755 --- a/tests/test-admin-deploy-grub2.sh +++ b/tests/test-admin-deploy-grub2.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..18" +echo "1..16" . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-syslinux.sh b/tests/test-admin-deploy-syslinux.sh index 797836f0..70b3b4d3 100755 --- a/tests/test-admin-deploy-syslinux.sh +++ b/tests/test-admin-deploy-syslinux.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..18" +echo "1..16" . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-uboot.sh b/tests/test-admin-deploy-uboot.sh index d9104f8c..d4c3a0db 100755 --- a/tests/test-admin-deploy-uboot.sh +++ b/tests/test-admin-deploy-uboot.sh @@ -20,7 +20,7 @@ set -euo pipefail -echo "1..19" +echo "1..17" . $(dirname $0)/libtest.sh From c651982929ec142d107e87d9f32ca324dffd4772 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Fri, 2 Jun 2017 08:30:43 -0700 Subject: [PATCH 37/86] Remove the OSTREE_MAX_RECURSION limit on metadata depth This was making it impossible to pull or mirror a large ostree repo, and according to Colin is no longer necessary. It works fine with a test against a repo with 2741 commit and 451468 objects in it. Closes: #899 Closes: #904 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 - src/libostree/ostree-core.h | 7 ------- src/libostree/ostree-repo-pull.c | 12 +----------- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 56c5c94e..5e111a96 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -89,7 +89,6 @@ OSTREE_VERSION_HEX ostree-core OSTREE_MAX_METADATA_SIZE OSTREE_MAX_METADATA_WARN_SIZE -OSTREE_MAX_RECURSION OstreeObjectType OSTREE_OBJECT_TYPE_IS_META OSTREE_OBJECT_TYPE_LAST diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index c530cea3..c1e014e2 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -44,13 +44,6 @@ G_BEGIN_DECLS */ #define OSTREE_MAX_METADATA_WARN_SIZE (7 * 1024 * 1024) -/** - * OSTREE_MAX_RECURSION: - * - * Maximum depth of metadata. - */ -#define OSTREE_MAX_RECURSION (256) - /** * OSTREE_SHA256_DIGEST_LEN: * diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 751129e6..536ba58f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -152,7 +152,7 @@ typedef struct { guchar csum[OSTREE_SHA256_DIGEST_LEN]; char *path; OstreeObjectType objtype; - guint recursion_depth; + guint recursion_depth; /* NB: not used anymore, though might be nice to print */ } ScanObjectQueueData; static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch); @@ -564,9 +564,6 @@ scan_dirtree_object (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - if (recursion_depth > OSTREE_MAX_RECURSION) - return glnx_throw (error, "Exceeded maximum recursion"); - g_autoptr(GVariant) tree = NULL; if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_DIR_TREE, checksum, &tree, error)) @@ -1188,13 +1185,6 @@ scan_commit_object (OtPullData *pull_data, gint depth; gboolean is_partial; - if (recursion_depth > OSTREE_MAX_RECURSION) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Exceeded maximum recursion"); - goto out; - } - if (g_hash_table_lookup_extended (pull_data->commit_to_depth, checksum, NULL, &depthp)) { From 3ec2b5773ea1553a70c362c25574978b7bbc932a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 2 Jun 2017 10:06:50 -0400 Subject: [PATCH 38/86] checkout: don't apply SELinux labeling in user mode If the user requested a user checkout, we don't want to set the SELinux label xattr. Closes: #903 Approved by: cgwalters --- src/libostree/ostree-repo-checkout.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 360c939f..8dbe49e3 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -261,14 +261,14 @@ create_file_copy_from_input_at (OstreeRepo *repo, &tmpf, error)) return FALSE; - if (sepolicy_enabled) + if (sepolicy_enabled && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { g_autofree char *label = NULL; - if (!ostree_sepolicy_get_label (options->sepolicy, - state->selabel_path_buf->str, + if (!ostree_sepolicy_get_label (options->sepolicy, state->selabel_path_buf->str, g_file_info_get_attribute_uint32 (file_info, "unix::mode"), &label, cancellable, error)) return FALSE; + if (fsetxattr (tmpf.fd, "security.selinux", label, strlen (label), 0) < 0) return glnx_throw_errno_prefix (error, "Setting security.selinux"); } From a32c6d2c70c82b583e8ee7c2a2f4e00b0d888fe5 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 2 Jun 2017 10:09:23 -0400 Subject: [PATCH 39/86] checkout: also chmod in the user checkout case When falling back to copying, we previously would only chmod checked out files in the non-user-checkout mode. Fix this by always doing chmod. The file_mode was being prepared but never actually applied. Add a basic test in the archive-z2 --> usermode checkout case in which we're guaranteed to always fall back to copy mode. Closes: #633 Closes: #903 Approved by: cgwalters --- src/libostree/ostree-repo-checkout.c | 22 +++++++++++----------- tests/basic-test.sh | 11 +++++++++-- tests/libtest-core.sh | 7 +++++++ tests/libtest.sh | 8 ++++++++ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 8dbe49e3..bb7c1771 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -140,10 +140,7 @@ write_regular_file_content (OstreeRepo *self, { if (TEMP_FAILURE_RETRY (fchown (outfd, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), g_file_info_get_attribute_uint32 (file_info, "unix::gid"))) < 0) - return glnx_throw_errno (error); - - if (TEMP_FAILURE_RETRY (fchmod (outfd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"))) < 0) - return glnx_throw_errno (error); + return glnx_throw_errno_prefix (error, "fchown"); if (xattrs) { @@ -152,10 +149,19 @@ write_regular_file_content (OstreeRepo *self, } } + guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); + + /* Don't make setuid files on checkout when we're doing --user */ + if (mode == OSTREE_REPO_CHECKOUT_MODE_USER) + file_mode &= ~(S_ISUID|S_ISGID); + + if (TEMP_FAILURE_RETRY (fchmod (outfd, file_mode)) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + if (fsync_is_enabled (self, options)) { if (fsync (outfd) == -1) - return glnx_throw_errno (error); + return glnx_throw_errno_prefix (error, "fsync"); } if (outstream) @@ -249,14 +255,8 @@ create_file_copy_from_input_at (OstreeRepo *repo, else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { g_auto(OtTmpfile) tmpf = { 0, }; - guint32 file_mode; GLnxLinkTmpfileReplaceMode replace_mode; - file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); - /* Don't make setuid files on checkout when we're doing --user */ - if (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) - file_mode &= ~(S_ISUID|S_ISGID); - if (!ot_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, &tmpf, error)) return FALSE; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index b209b839..a393d7fc 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -28,7 +28,7 @@ echo "ok yaml version" CHECKOUT_U_ARG="" COMMIT_ARGS="" DIFF_ARGS="" -if grep -q bare-user-only repo/config; then +if is_bare_user_only_repo repo; then # In bare-user-only repos we can only represent files with uid/gid 0, no # xattrs and canonical permissions, so we need to commit them as such, or # we end up with repos that don't pass fsck @@ -66,7 +66,7 @@ validate_checkout_basic checkout-test2 rm checkout-test2 -rf # Only do these tests on bare-user/bare, not bare-user-only # since the latter automatically synthesizes -U if it's not passed. -if ! grep -q bare-user-only repo/config; then +if ! is_bare_user_only_repo repo; then if grep -q bare-user repo/config; then if $OSTREE checkout -H test2 checkout-test2 2>err.txt; then assert_not_reached "checkout -H worked?" @@ -566,6 +566,13 @@ rm test2-checkout -rf ${CMD_PREFIX} ostree --repo=repo2 checkout -U test2 test2-checkout assert_file_has_content test2-checkout/baz/cow moo assert_has_dir repo2/uncompressed-objects-cache +# we're in archive mode, but the repo we pull-local from might be +# bare-user-only, in which case, we skip these checks since bare-user-only +# doesn't store permission bits +if ! is_bare_user_only_repo repo; then + assert_file_has_mode test2-checkout/baz/cowro 600 + assert_file_has_mode test2-checkout/baz/deeper/ohyeahx 755 +fi echo "ok disable cache checkout" cd ${test_tmpdir} diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index 14d9cd5c..de850544 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -110,6 +110,13 @@ assert_file_has_content_literal () { fi } +assert_file_has_mode () { + mode=$(stat -c '%a' $1) + if [ "$mode" != "$2" ]; then + fatal "File '$1' has wrong mode: expected $2, but got $mode" + fi +} + assert_symlink_has_content () { if ! test -L "$1"; then fatal "File '$1' is not a symbolic link" diff --git a/tests/libtest.sh b/tests/libtest.sh index 1a81c755..f1fba4fc 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -157,9 +157,13 @@ setup_test_repository () { mkdir baz echo moo > baz/cow + echo mooro > baz/cowro + chmod 600 baz/cowro echo alien > baz/saucer mkdir baz/deeper echo hi > baz/deeper/ohyeah + echo hix > baz/deeper/ohyeahx + chmod 755 baz/deeper/ohyeahx ln -s nonexistent baz/alink mkdir baz/another/ echo x > baz/another/y @@ -495,3 +499,7 @@ has_gpgme () { libtest_cleanup_gpg () { gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye || true } + +is_bare_user_only_repo () { + grep -q 'mode=bare-user-only' $1/config +} From ab7c3fd8003220304717aa8a951ec64b1ebf8182 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 2 Jun 2017 13:18:49 -0400 Subject: [PATCH 40/86] manual: document bare-user-only repo mode Closes: #903 Approved by: cgwalters --- docs/manual/repo.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index 6e307ba3..6b547be6 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -81,6 +81,13 @@ The `bare-user` mode is useful for build systems that run as non-root but want to generate root-owned content, as well as non-root container systems. +There is a variant to the `bare-user` mode called `bare-user-only`. Unlike +`bare-user`, neither ownership nor extended attributes are stored. These repos +are meant to to be checked out in user mode (with the `-U` flag), where this +information is not applied anyway. The main advantage of `bare-user-only` is +that repos can be stored on filesystems which do not support extended +attributes, such as tmpfs. + In contrast, the `archive-z2` mode is designed for serving via plain HTTP. Like tar files, it can be read/written by non-root users. From f813ae74addef8784469a33f0b37c63f05596555 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 2 Jun 2017 13:41:33 -0400 Subject: [PATCH 41/86] basic-test.sh: explicitly check for uncompressed objects It's not enough to check that the dir exists, since that's done by default when we open the repo. We want to actually check that uncompressed objects were cached (i.e. the opposite of the earlier error path). Closes: #903 Approved by: cgwalters --- tests/basic-test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index a393d7fc..d56bcc62 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -566,6 +566,10 @@ rm test2-checkout -rf ${CMD_PREFIX} ostree --repo=repo2 checkout -U test2 test2-checkout assert_file_has_content test2-checkout/baz/cow moo assert_has_dir repo2/uncompressed-objects-cache +ls repo2/uncompressed-objects-cache > ls.txt +if ! test -s ls.txt; then + assert_not_reached "repo didn't cache uncompressed objects" +fi # we're in archive mode, but the repo we pull-local from might be # bare-user-only, in which case, we skip these checks since bare-user-only # doesn't store permission bits From e18cacb06e9d4381094a68001f6b6f68509be8e9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 2 Jun 2017 15:26:26 -0400 Subject: [PATCH 42/86] Don't install trivial-httpd man page if not enabled I just noticed this scroll by in a file listing. Closes: #905 Approved by: jlebon --- Makefile-man.am | 5 ++++- configure.ac | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile-man.am b/Makefile-man.am index bdc78947..7996d2d5 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -29,7 +29,10 @@ ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 \ ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 \ ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 \ ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 \ -ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 +ostree-summary.1 ostree-static-delta.1 +if BUILDOPT_TRIVIAL_HTTPD +man1_files += ostree-trivial-httpd.1 +endif if BUILDOPT_FUSE man1_files += rofiles-fuse.1 diff --git a/configure.ac b/configure.ac index a6abee85..c60806f4 100644 --- a/configure.ac +++ b/configure.ac @@ -148,7 +148,8 @@ AC_ARG_ENABLE(trivial-httpd-cmdline, [AS_HELP_STRING([--enable-trivial-httpd-cmdline], [Continue to support "ostree trivial-httpd" [default=no]])],, enable_trivial_httpd_cmdline=no) -AS_IF([test x$enable_trivial_httpd_cmdline = xyes], +AM_CONDITIONAL(BUILDOPT_TRIVIAL_HTTPD, test x$enable_trivial_httpd_cmdline = xyes) +AM_COND_IF(BUILDOPT_TRIVIAL_HTTPD, [AC_DEFINE([BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE], 1, [Define if we are enabling ostree trivial-httpd entrypoint])] ) From 25696b3fb064bd933d1e05e8175e6d95891fa2c0 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Tue, 6 Jun 2017 22:51:00 -0400 Subject: [PATCH 43/86] lib/repo: Don't copy xattrs when manipulating the GPG keyring Copying xattrs when manipulating the GPG keyring for a repository causes errors when the underlying filesystem doesn't support writing xattrs - overlayfs is a common example. It also causes the selinux attributes of the keyring files to be copied from the temporary location instead of properly inherited from the destination directory (ending up, for example, as unconfined_u:object_r:user_tmp_t:s0, rather than unconfined_u:object_r:data_home_t:s0) Closes: #910 Approved by: cgwalters --- src/libostree/ostree-repo.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index cbbaec9b..abdb63c1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1329,7 +1329,6 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, struct stat stbuf; gpgme_error_t gpg_error; gboolean ret = FALSE; - const GLnxFileCopyFlags copyflags = self->disable_xattrs ? GLNX_FILE_COPY_NOXATTRS : 0; g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); @@ -1453,7 +1452,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, { if (!glnx_file_copy_at (self->repo_dir_fd, remote->keyring, &stbuf, target_temp_fd, "pubring.gpg", - copyflags, cancellable, error)) + GLNX_FILE_COPY_NOXATTRS, cancellable, error)) { g_prefix_error (error, "Unable to copy remote's keyring: "); goto out; @@ -1537,7 +1536,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, * updated keyring in the target context's temporary directory. */ if (!glnx_file_copy_at (target_temp_fd, "pubring.gpg", NULL, self->repo_dir_fd, remote->keyring, - copyflags | GLNX_FILE_COPY_OVERWRITE, + GLNX_FILE_COPY_NOXATTRS | GLNX_FILE_COPY_OVERWRITE, cancellable, error)) goto out; From 0c4b3a2b6da950fd78e63f9afec602f6188f1ab0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 6 Jun 2017 13:34:27 -0400 Subject: [PATCH 44/86] Canonicalize bare-user-only perms with 0755 mask For the flatpak use case where bare-user-only was introduced, we actually don't want to support s{u,g} id files in particular. Actually, I can't think of a reason to have anything outside of the `0755 i.e. (u=rwx,g=rx,o=rx)` mask, so that's what we do here. This will have the effect of treating existing `bare-user-only` repositories as corrupted if they have files outside of that mask, but I think we should do this now; most of the flatpak users will still be on `bare-user`, and we haven't changed the semantics of that mode yet. Note that in this patch we will also *reject* file content that doesn't match this. This is somewhat asymmetric, since we aren't similarly rejecting e.g. directory metadata. But, this will close off the biggest source of the problem for flatpak (setuid binaries). See: https://github.com/ostreedev/ostree/pull/908 See: https://github.com/flatpak/flatpak/pull/837 Closes: #909 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 37 ++++++++++++++++++++++++------ tests/basic-test.sh | 9 ++++++-- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 870a06c7..3ecea29d 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -281,7 +281,7 @@ commit_loose_object_trusted (OstreeRepo *self, } if (objtype == OSTREE_OBJECT_TYPE_FILE && - (self->mode == OSTREE_REPO_MODE_BARE_USER || self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)) + self->mode == OSTREE_REPO_MODE_BARE_USER) { if (!object_is_symlink) { @@ -294,10 +294,21 @@ commit_loose_object_trusted (OstreeRepo *self, return glnx_throw_errno (error); } - if (self->mode == OSTREE_REPO_MODE_BARE_USER && - !write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) + if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) return FALSE; } + else if (objtype == OSTREE_OBJECT_TYPE_FILE && + self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY + && !object_is_symlink) + { + guint32 invalid_modebits = (mode & ~S_IFMT) & ~0755; + if (invalid_modebits > 0) + return glnx_throw (error, "Invalid mode 0%04o with bits 0%04o in bare-user-only repository", + mode, invalid_modebits); + + if (fchmod (fd, mode) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + } if (objtype == OSTREE_OBJECT_TYPE_FILE && _ostree_repo_mode_is_bare (self->mode)) { @@ -2293,11 +2304,23 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, if ((modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS) != 0) { - - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + guint mode = g_file_info_get_attribute_uint32 (modified_info, "unix::mode"); + switch (g_file_info_get_file_type (file_info)) { - guint current_mode = g_file_info_get_attribute_uint32 (modified_info, "unix::mode"); - g_file_info_set_attribute_uint32 (modified_info, "unix::mode", current_mode | 0744); + case G_FILE_TYPE_REGULAR: + /* In particular, we want to squash the s{ug}id bits, but this also + * catches the sticky bit for example. + */ + g_file_info_set_attribute_uint32 (modified_info, "unix::mode", mode & (S_IFREG | 0755)); + break; + case G_FILE_TYPE_DIRECTORY: + /* Like the above but for directories */ + g_file_info_set_attribute_uint32 (modified_info, "unix::mode", mode & (S_IFDIR | 0755)); + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + break; + default: + g_assert_not_reached (); } g_file_info_set_attribute_uint32 (modified_info, "unix::uid", 0); g_file_info_set_attribute_uint32 (modified_info, "unix::gid", 0); diff --git a/tests/basic-test.sh b/tests/basic-test.sh index d56bcc62..82945244 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -312,8 +312,13 @@ cd ${test_tmpdir}/checkout-test2-4 $OSTREE commit ${COMMIT_ARGS} -b test2-override -s "with statoverride" --statoverride=../test-statoverride.txt cd ${test_tmpdir} $OSTREE checkout test2-override checkout-test2-override -test -g checkout-test2-override/a/nested/2 -test -u checkout-test2-override/a/nested/3 +if ! is_bare_user_only_repo repo; then + test -g checkout-test2-override/a/nested/2 + test -u checkout-test2-override/a/nested/3 +else + test '!' -g checkout-test2-override/a/nested/2 + test '!' -u checkout-test2-override/a/nested/3 +fi echo "ok commit statoverride" cd ${test_tmpdir} From 12479d8b051631c630e8ec090a200da59a277cf5 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 31 May 2017 14:46:40 +0100 Subject: [PATCH 45/86] lib/repo: Reindent some code in regenerate_summary() for clarity This makes it a bit more easily separable from the rest of the code in the function. No functional changes. Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo.c | 69 +++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index abdb63c1..c1aedb91 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4476,54 +4476,55 @@ ostree_repo_regenerate_summary (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GHashTable) refs = NULL; - if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) - return FALSE; - g_auto(GVariantDict) additional_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; g_variant_dict_init (&additional_metadata_builder, additional_metadata); g_autoptr(GVariantBuilder) refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); - g_autoptr(GList) ordered_keys = g_hash_table_get_keys (refs); - ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); + { + g_autoptr(GHashTable) refs = NULL; + if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) + return FALSE; - for (GList *iter = ordered_keys; iter; iter = iter->next) - { - const char *ref = iter->data; - const char *commit = g_hash_table_lookup (refs, ref); - g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_autoptr(GList) ordered_keys = g_hash_table_get_keys (refs); + ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); - g_assert (commit); + for (GList *iter = ordered_keys; iter; iter = iter->next) + { + const char *ref = iter->data; + const char *commit = g_hash_table_lookup (refs, ref); + g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_autofree char *remotename = NULL; - if (!ostree_parse_refspec (ref, &remotename, NULL, NULL)) - g_assert_not_reached (); + g_assert (commit); - /* Don't put remote refs in the summary */ - if (remotename != NULL) - continue; + g_autofree char *remotename = NULL; + if (!ostree_parse_refspec (ref, &remotename, NULL, NULL)) + g_assert_not_reached (); - g_autoptr(GVariant) commit_obj = NULL; - if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) - return FALSE; + /* Don't put remote refs in the summary */ + if (remotename != NULL) + continue; - g_variant_dict_init (&commit_metadata_builder, NULL); + g_autoptr(GVariant) commit_obj = NULL; + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) + return FALSE; - /* Forward the commit’s timestamp if it’s valid. */ - guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); - g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); + g_variant_dict_init (&commit_metadata_builder, NULL); - if (dt != NULL) - g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, - g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + /* Forward the commit’s timestamp if it’s valid. */ + guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); - g_variant_builder_add_value (refs_builder, - g_variant_new ("(s(t@ay@a{sv}))", ref, - (guint64) g_variant_get_size (commit_obj), - ostree_checksum_to_bytes_v (commit), - g_variant_dict_end (&commit_metadata_builder))); - } + if (dt != NULL) + g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, + g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + g_variant_builder_add_value (refs_builder, + g_variant_new ("(s(t@ay@a{sv}))", ref, + (guint64) g_variant_get_size (commit_obj), + ostree_checksum_to_bytes_v (commit), + g_variant_dict_end (&commit_metadata_builder))); + } + } { g_autoptr(GPtrArray) delta_names = NULL; From 52ede69df8e27a28a5e941fa9769405141c914ca Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:46:02 +0100 Subject: [PATCH 46/86] lib/pull: Fix a typo in a documentation comment Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 536ba58f..0585eb91 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2690,7 +2690,7 @@ initiate_request (OtPullData *pull_data, * * override-commit-ids (as): Array of specific commit IDs to fetch for refs * * dry-run (b): Only print information on what will be downloaded (requires static deltas) * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options - * * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do multiple pulls in one transaction. + * * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid */ From 14082e6b030e11391c429c3e3b878ada717affdd Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:46:15 +0100 Subject: [PATCH 47/86] lib/pull: Simplify a for-loop initialisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s a bit neater to initialise the loop iterator and maximum in the same place. Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0585eb91..d1826117 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3174,8 +3174,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->summary) { refs = g_variant_get_child_value (pull_data->summary, 0); - n = g_variant_n_children (refs); - for (i = 0; i < n; i++) + for (i = 0, n = g_variant_n_children (refs); i < n; i++) { const char *refname; g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i); From c968c12ec34af831b148aea14364406c5d383196 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:28:23 +0100 Subject: [PATCH 48/86] lib/pull: Drop some trailing whitespace Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index d1826117..5d54a5a1 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3241,7 +3241,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid); } - + strviter++; if (commitid_strviter) commitid_strviter++; @@ -3377,7 +3377,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, ret = TRUE; goto out; } - + g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0); g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0); g_assert_cmpint (pull_data->n_outstanding_content_fetches, ==, 0); @@ -3390,7 +3390,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, const char *checksum = value; g_autofree char *remote_ref = NULL; g_autofree char *original_rev = NULL; - + if (pull_data->remote_name) remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); else From 87413ee63b1612ee4d870e2e19b2b58643201641 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:31:19 +0100 Subject: [PATCH 49/86] lib/pull: Fix an over-indented block Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5d54a5a1..ed782cd5 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3479,15 +3479,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0)) goto out; } - g_hash_table_iter_init (&hash_iter, commits_to_fetch); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) - { - const char *commit = value; - g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit); - if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0)) - goto out; - } + g_hash_table_iter_init (&hash_iter, commits_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *commit = value; + g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit); + + if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0)) + goto out; + } } ret = TRUE; From 7ce6777028953457c1f4125c48e6dc99a37949fc Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 12:03:57 +0100 Subject: [PATCH 50/86] ostree/dump: Improve formatting for well-known commit metadata keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This follows on from commit a946c3d4, which added formatting for well-known summary metadata keys. This commit adds it for commits. Currently, the only well-known commit metadata key is ostree.commit.timestamp. Formatting this correctly is especially important, since it’s a big-endian uint64, which is completely unusable for mere mortals when presented as a number rather than a date. Non-formatted output can still be retrieved using the OSTREE_DUMP_RAW flag, and the non-formatted key name is always printed for clarity. Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/ostree/ot-dump.c | 44 ++++++++++++++++++++++++++------------ tests/test-summary-view.sh | 1 + 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index b24003c7..e6b8859b 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -71,6 +71,18 @@ format_timestamp (guint64 timestamp, return str; } +static gchar * +uint64_secs_to_iso8601 (guint64 secs) +{ + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); + g_autoptr(GDateTime) local = (dt != NULL) ? g_date_time_to_local (dt) : NULL; + + if (local != NULL) + return g_date_time_format (local, "%FT%T%:::z"); + else + return g_strdup ("invalid"); +} + static void dump_indented_lines (const gchar *data) { @@ -196,23 +208,27 @@ dump_summary_ref (const char *ref_name, while (g_variant_iter_loop (metadata, "{sv}", &key, &value)) { - g_autofree char *string = g_variant_print (value, FALSE); - g_print (" %s: %s\n", key, string); + g_autofree gchar *value_str = NULL; + const gchar *pretty_key = NULL; + + if (g_strcmp0 (key, OSTREE_COMMIT_TIMESTAMP) == 0) + { + pretty_key = "Timestamp"; + value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); + } + else + { + value_str = g_variant_print (value, FALSE); + } + + /* Print out. */ + if (pretty_key != NULL) + g_print (" %s (%s): %s\n", pretty_key, key, value_str); + else + g_print (" %s: %s\n", key, value_str); } } -static gchar * -uint64_secs_to_iso8601 (guint64 secs) -{ - g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); - g_autoptr(GDateTime) local = (dt != NULL) ? g_date_time_to_local (dt) : NULL; - - if (local != NULL) - return g_date_time_format (local, "%FT%T%:::z"); - else - return g_strdup ("invalid"); -} - void ot_dump_summary_bytes (GBytes *summary_bytes, OstreeDumpFlags flags) diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh index 6dcfd088..5b52b691 100755 --- a/tests/test-summary-view.sh +++ b/tests/test-summary-view.sh @@ -52,6 +52,7 @@ assert_file_has_content_literal summary.txt "* main" assert_file_has_content_literal summary.txt "* other" assert_file_has_content_literal summary.txt "ostree.summary.last-modified" assert_file_has_content_literal summary.txt "Static Deltas (ostree.static-deltas): {}" +assert_file_has_content_literal summary.txt "Timestamp (ostree.commit.timestamp): " echo "ok view summary" # Check the summary can be viewed raw too. From 59ffce73c8c386f70c850f318934f1581ef05e7a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:32:29 +0100 Subject: [PATCH 51/86] lib/repo: Omit deltas from the summary file if there are none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If there are no deltas to be listed in the summary file, don’t bother including the key for them in the additional metadata section of the file. This saves a few bytes in some cases. Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-repo.c | 3 ++- tests/test-summary-view.sh | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index c1aedb91..0d20c25b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4561,7 +4561,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], ot_gvariant_new_bytearray (csum, 32)); } - g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); + if (delta_names->len > 0) + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); } { diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh index 5b52b691..60855eb1 100755 --- a/tests/test-summary-view.sh +++ b/tests/test-summary-view.sh @@ -51,7 +51,6 @@ ${OSTREE} summary --view > summary.txt assert_file_has_content_literal summary.txt "* main" assert_file_has_content_literal summary.txt "* other" assert_file_has_content_literal summary.txt "ostree.summary.last-modified" -assert_file_has_content_literal summary.txt "Static Deltas (ostree.static-deltas): {}" assert_file_has_content_literal summary.txt "Timestamp (ostree.commit.timestamp): " echo "ok view summary" From 4418ab7fa96ae45d330697b697376ae121c09b0b Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 15:41:17 +0100 Subject: [PATCH 52/86] lib/fetcher: Add cleanup function for OstreeFetcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is only used internally (the header is not public), so it doesn’t have to go in ostree-autocleanups.h. It will be used in some following commits. Signed-off-by: Philip Withnall Closes: #911 Approved by: cgwalters --- src/libostree/ostree-fetcher.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 16cf3d7d..8ec5f209 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -46,6 +46,8 @@ struct OstreeFetcherClass GObjectClass parent_class; }; +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcher, g_object_unref) + typedef enum { OSTREE_FETCHER_FLAGS_NONE = 0, OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0), From 6b402e53f4b111500e258f52735b66569ac96929 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 14:45:42 -0400 Subject: [PATCH 53/86] builtins/cat: Port to new code style Definitely better. Prep for another fix. Closes: #915 Approved by: jlebon --- src/ostree/ot-builtin-cat.c | 57 +++++++++++++------------------------ 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/src/ostree/ot-builtin-cat.c b/src/ostree/ot-builtin-cat.c index a784afe1..db0ffab3 100644 --- a/src/ostree/ot-builtin-cat.c +++ b/src/ostree/ot-builtin-cat.c @@ -39,64 +39,45 @@ cat_one_file (GFile *f, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GInputStream) in = NULL; - - in = (GInputStream*)g_file_read (f, cancellable, error); + g_autoptr(GInputStream) in = (GInputStream*)g_file_read (f, cancellable, error); if (!in) - goto out; + return FALSE; - { - gssize n_bytes_written = g_output_stream_splice (stdout_stream, in, G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, - cancellable, error); - if (n_bytes_written < 0) - goto out; - } + if (g_output_stream_splice (stdout_stream, in, G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE, + cancellable, error) < 0) + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } gboolean ostree_builtin_cat (int argc, char **argv, GCancellable *cancellable, GError **error) { - g_autoptr(GOptionContext) context = NULL; - glnx_unref_object OstreeRepo *repo = NULL; - gboolean ret = FALSE; - int i; - const char *rev; - g_autoptr(GOutputStream) stdout_stream = NULL; - g_autoptr(GFile) root = NULL; - g_autoptr(GFile) f = NULL; - - context = g_option_context_new ("COMMIT PATH... - Concatenate contents of files"); - + g_autoptr(GOptionContext) context = g_option_context_new ("COMMIT PATH... - Concatenate contents of files"); + g_autoptr(OstreeRepo) repo = NULL; if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) - goto out; + return FALSE; if (argc <= 2) { ot_util_usage_error (context, "A COMMIT and at least one PATH argument are required", error); - goto out; + return FALSE; } - rev = argv[1]; + const char *rev = argv[1]; + g_autoptr(GFile) root = NULL; if (!ostree_repo_read_commit (repo, rev, &root, NULL, NULL, error)) - goto out; + return FALSE; - stdout_stream = g_unix_output_stream_new (1, FALSE); + g_autoptr(GOutputStream) stdout_stream = g_unix_output_stream_new (1, FALSE); - for (i = 2; i < argc; i++) + for (int i = 2; i < argc; i++) { - g_clear_object (&f); - f = g_file_resolve_relative_path (root, argv[i]); + g_autoptr(GFile) f = g_file_resolve_relative_path (root, argv[i]); if (!cat_one_file (f, stdout_stream, cancellable, error)) - goto out; + return FALSE; } - - ret = TRUE; - out: - return ret; + + return TRUE; } From 807a804b1688db62ac552e58cabf55a55ca84071 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 14:53:06 -0400 Subject: [PATCH 54/86] lib/repofile: Port mostly to new code style Prep for a bugfix. Closes: #915 Approved by: jlebon --- src/libostree/ostree-repo-file.c | 119 +++++++++++-------------------- 1 file changed, 41 insertions(+), 78 deletions(-) diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c index a88e5363..2a90f6c7 100644 --- a/src/libostree/ostree-repo-file.c +++ b/src/libostree/ostree-repo-file.c @@ -148,7 +148,6 @@ _ostree_repo_file_new_for_commit (OstreeRepo *repo, const char *commit, GError **error) { - OstreeRepoFile *ret = NULL; g_autoptr(GVariant) commit_v = NULL; g_autoptr(GVariant) tree_contents_csum_v = NULL; g_autoptr(GVariant) tree_metadata_csum_v = NULL; @@ -161,7 +160,7 @@ _ostree_repo_file_new_for_commit (OstreeRepo *repo, if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_v, error)) - goto out; + return NULL; /* PARSE OSTREE_OBJECT_TYPE_COMMIT */ g_variant_get_child (commit_v, 6, "@ay", &tree_contents_csum_v); @@ -172,17 +171,13 @@ _ostree_repo_file_new_for_commit (OstreeRepo *repo, ostree_checksum_inplace_from_bytes (g_variant_get_data (tree_metadata_csum_v), tree_metadata_csum); - ret = _ostree_repo_file_new_root (repo, tree_contents_csum, tree_metadata_csum); - - out: - return ret; + return _ostree_repo_file_new_root (repo, tree_contents_csum, tree_metadata_csum); } static gboolean do_resolve (OstreeRepoFile *self, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) root_contents = NULL; g_autoptr(GVariant) root_metadata = NULL; @@ -190,27 +185,24 @@ do_resolve (OstreeRepoFile *self, if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_DIR_TREE, self->tree_contents_checksum, &root_contents, error)) - goto out; + return FALSE; if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_DIR_META, self->tree_metadata_checksum, &root_metadata, error)) - goto out; - + return FALSE; + self->tree_metadata = root_metadata; root_metadata = NULL; self->tree_contents = root_contents; root_contents = NULL; - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean do_resolve_nonroot (OstreeRepoFile *self, GError **error) { - gboolean ret = FALSE; gboolean is_dir; int i; g_autoptr(GVariant) container = NULL; @@ -221,13 +213,13 @@ do_resolve_nonroot (OstreeRepoFile *self, g_autofree char *tmp_checksum = NULL; if (!ostree_repo_file_ensure_resolved (self->parent, error)) - goto out; + return FALSE; if (!self->parent->tree_contents) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_DIRECTORY, "Not a directory"); - goto out; + return FALSE; } i = ostree_repo_file_tree_find_child (self->parent, self->name, &is_dir, &container); @@ -235,7 +227,7 @@ do_resolve_nonroot (OstreeRepoFile *self, if (i < 0) { set_error_noent ((GFile*)self, error); - goto out; + return FALSE; } if (is_dir) @@ -255,14 +247,14 @@ do_resolve_nonroot (OstreeRepoFile *self, if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_DIR_TREE, tmp_checksum, &tree_contents, error)) - goto out; + return FALSE; g_free (tmp_checksum); tmp_checksum = ostree_checksum_from_bytes_v (metadata_csum_v); if (!ostree_repo_load_variant (self->repo, OSTREE_OBJECT_TYPE_DIR_META, tmp_checksum, &tree_metadata, error)) - goto out; + return FALSE; self->tree_contents = tree_contents; tree_contents = NULL; @@ -274,35 +266,29 @@ do_resolve_nonroot (OstreeRepoFile *self, else self->index = i; - ret = TRUE; - out: - return ret; + return TRUE; } gboolean ostree_repo_file_ensure_resolved (OstreeRepoFile *self, GError **error) { - gboolean ret = FALSE; - if (self->parent == NULL) { if (self->tree_contents == NULL) if (!do_resolve (self, error)) - goto out; + return FALSE; } else { if (self->index == -1) { if (!do_resolve_nonroot (self, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -311,25 +297,21 @@ ostree_repo_file_get_xattrs (OstreeRepoFile *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GVariant) ret_xattrs = NULL; - if (!ostree_repo_file_ensure_resolved (self, error)) - goto out; + return FALSE; + g_autoptr(GVariant) ret_xattrs = NULL; if (self->tree_metadata) ret_xattrs = g_variant_get_child_value (self->tree_metadata, 3); - else + else { if (!ostree_repo_load_file (self->repo, ostree_repo_file_get_checksum (self), NULL, NULL, &ret_xattrs, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; ot_transfer_out_value(out_xattrs, &ret_xattrs); - out: - return ret; + return TRUE; } GVariant * @@ -721,28 +703,24 @@ query_child_info_dir (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GFileInfo) ret_info = NULL; - g_autoptr(GVariant) metadata = NULL; - ret_info = g_file_info_new (); + g_autoptr(GFileInfo) ret_info = g_file_info_new (); g_file_info_set_attribute_uint32 (ret_info, "standard::type", G_FILE_TYPE_DIRECTORY); - + if (g_file_attribute_matcher_matches (matcher, "unix::mode")) { + g_autoptr(GVariant) metadata = NULL; if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_DIR_META, metadata_checksum, &metadata, error)) - goto out; + return FALSE; set_info_from_dirmeta (ret_info, metadata); } - - ret = TRUE; + ot_transfer_out_value(out_info, &ret_info); - out: - return ret; + return TRUE; } int @@ -890,12 +868,11 @@ ostree_repo_file_query_info (GFile *file, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; OstreeRepoFile *self = OSTREE_REPO_FILE (file); g_autoptr(GFileInfo) info = NULL; if (!ostree_repo_file_ensure_resolved (self, error)) - goto out; + return NULL; if (!self->parent) { @@ -904,20 +881,14 @@ ostree_repo_file_query_info (GFile *file, } else { - if (!ostree_repo_file_tree_query_child (self->parent, self->index, - attributes, flags, + if (!ostree_repo_file_tree_query_child (self->parent, self->index, + attributes, flags, &info, cancellable, error)) - goto out; + return NULL; g_assert (info != NULL); } - - ret = TRUE; - out: - if (!ret) - g_clear_object (&info); - else - g_object_ref (info); - return info; + + return g_steal_pointer (&info); } static GFileAttributeInfoList * @@ -938,38 +909,30 @@ ostree_repo_file_query_writable_namespaces (GFile *file, static GFileInputStream * ostree_repo_file_read (GFile *file, - GCancellable *cancellable, - GError **error) + GCancellable *cancellable, + GError **error) { - gboolean ret = FALSE; OstreeRepoFile *self = OSTREE_REPO_FILE (file); const char *checksum; g_autoptr(GInputStream) ret_stream = NULL; if (!ostree_repo_file_ensure_resolved (self, error)) - goto out; + return FALSE; if (self->tree_contents) { - g_set_error_literal (error, G_IO_ERROR, - G_IO_ERROR_IS_DIRECTORY, - "Can't open directory"); - goto out; + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_IS_DIRECTORY, + "Can't open directory"); + return NULL; } checksum = ostree_repo_file_get_checksum (self); if (!ostree_repo_load_file (self->repo, checksum, &ret_stream, NULL, NULL, cancellable, error)) - goto out; - - ret = TRUE; - out: - if (!ret) - g_clear_object (&ret_stream); - else - g_object_ref (ret_stream); - return (GFileInputStream*)ret_stream; + return NULL; + + return g_steal_pointer (&ret_stream); } static void From d3900f90f4ca3fbded4ef5a75856dc9036037348 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 15:25:21 -0400 Subject: [PATCH 55/86] lib/repofile: Follow symlinks for `g_file_read()` This avoids `ostree cat /path/to/symlink` crashing, a longstanding embarassing issue. Closes: #915 Approved by: jlebon --- src/libostree/ostree-repo-file.c | 18 ++++++++++++++++-- tests/basic-test.sh | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c index 2a90f6c7..fcc435a8 100644 --- a/src/libostree/ostree-repo-file.c +++ b/src/libostree/ostree-repo-file.c @@ -928,9 +928,23 @@ ostree_repo_file_read (GFile *file, checksum = ostree_repo_file_get_checksum (self); - if (!ostree_repo_load_file (self->repo, checksum, &ret_stream, - NULL, NULL, cancellable, error)) + g_autoptr(GFileInfo) finfo = NULL; + if (!ostree_repo_load_file (self->repo, checksum, NULL, + &finfo, NULL, cancellable, error)) return NULL; + if (g_file_info_get_file_type (finfo) == G_FILE_TYPE_REGULAR) + { + if (!ostree_repo_load_file (self->repo, checksum, &ret_stream, + NULL, NULL, cancellable, error)) + return NULL; + } + else + { + g_autoptr(GFile) parent = g_file_get_parent (file); + const char *target = g_file_info_get_symlink_target (finfo); + g_autoptr(GFile) dest = g_file_resolve_relative_path (parent, target); + return g_file_read (dest, cancellable, error); + } return g_steal_pointer (&ret_stream); } diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 82945244..f5af93b1 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -338,8 +338,30 @@ $OSTREE prune echo "ok prune didn't fail" cd ${test_tmpdir} +# Verify we can't cat dirs +for path in / /baz; do + if $OSTREE cat test2 $path 2>err.txt; then + assert_not_reached "cat directory" + fi + assert_file_has_content err.txt "open directory" +done +rm checkout-test2 -rf $OSTREE cat test2 /yet/another/tree/green > greenfile-contents assert_file_has_content greenfile-contents "leaf" +$OSTREE checkout test2 checkout-test2 +ls -alR checkout-test2 +ln -sr checkout-test2/{four,four-link} +ln -sr checkout-test2/{baz/cow,cow-link} +ln -sr checkout-test2/{cow-link,cow-link-link} +$OSTREE commit -b test2-withlink --tree=dir=checkout-test2 +if $OSTREE cat test2-withlink /four-link 2>err.txt; then + assert_not_reached "cat directory" +fi +assert_file_has_content err.txt "open directory" +for path in /cow-link /cow-link-link; do + $OSTREE cat test2-withlink $path >contents.txt + assert_file_has_content contents.txt moo +done echo "ok cat-file" cd ${test_tmpdir} From 5913b2294490a1f85d4b617fdcc9455ed0bc63a2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Jun 2017 11:32:52 -0400 Subject: [PATCH 56/86] lib/repo: For bare-user, mask content object modes with 0775 Having every object in a bare-user repo (and checkouts) be executable is ugly. I can't think of a good reason to do that; they should only be executable if their input is. This does for `bare-user` what we did for `bare-user-only` in https://github.com/ostreedev/ostree/pull/909 It's also a stronger version of what we do with `checkout -U` in suppressing suid - here we also strip world-writable files and the sticky bit (even though that's meaningless today, it might not be in the future). Closes: https://github.com/ostreedev/ostree/issues/907 Closes: #908 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 22 ++++++++--------- tests/basic-test.sh | 2 +- tests/libtest.sh | 18 ++++++++++++++ tests/test-basic-user.sh | 38 ++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 3ecea29d..da0a5cdc 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -283,19 +283,19 @@ commit_loose_object_trusted (OstreeRepo *self, if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER) { - if (!object_is_symlink) - { - /* We need to apply at least some mode bits, because the repo file was created - with mode 644, and we need e.g. exec bits to be right when we do a user-mode - checkout. To make this work we apply all user bits and the read bits for - group/other. Furthermore, setting user xattrs requires write access, so - this makes sure it's at least writable by us. (O_TMPFILE uses mode 0 by default) */ - if (fchmod (fd, mode | 0744) < 0) - return glnx_throw_errno (error); - } - if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) return FALSE; + + if (!object_is_symlink) + { + /* Note that previously this path added `| 0755` which made every + * file executable, see + * https://github.com/ostreedev/ostree/issues/907 + */ + const mode_t content_mode = (mode & (S_IFREG | 0775)); + if (fchmod (fd, content_mode) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + } } else if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY diff --git a/tests/basic-test.sh b/tests/basic-test.sh index f5af93b1..e3e442e9 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..66" +echo "1..$((66 + ${extra_basic_tests:-0}))" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' diff --git a/tests/libtest.sh b/tests/libtest.sh index f1fba4fc..1774a7b6 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -503,3 +503,21 @@ libtest_cleanup_gpg () { is_bare_user_only_repo () { grep -q 'mode=bare-user-only' $1/config } + +# Given a path to a file in a repo for a ref, print its checksum +ostree_file_path_to_checksum() { + repo=$1 + ref=$2 + path=$3 + $CMD_PREFIX ostree --repo=$repo ls -C $ref $path | awk '{ print $5 }' +} + +# Given a path to a file in a repo for a ref, print the path to its object +ostree_file_path_to_object_path() { + repo=$1 + ref=$2 + path=$3 + checksum=$(ostree_file_path_to_checksum $repo $ref $path) + test -n "${checksum}" + echo ${repo}/objects/${checksum:0:2}/${checksum:2}.file +} diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index 3e11545e..fa802df6 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -25,4 +25,42 @@ skip_without_user_xattrs setup_test_repository "bare-user" +extra_basic_tests=3 . $(dirname $0)/basic-test.sh + +# Reset things so we don't inherit a lot of state from earlier tests +rm repo files -rf +setup_test_repository "bare-user" + +cd ${test_tmpdir} +objpath_nonexec=$(ostree_file_path_to_object_path repo test2 baz/cow) +# Sigh, umask +touch testfile +default_mode=$(stat -c '%a' testfile) +rm testfile +assert_file_has_mode ${objpath_nonexec} ${default_mode} +objpath_ro=$(ostree_file_path_to_object_path repo test2 baz/cowro) +assert_file_has_mode ${objpath_ro} 600 +objpath_exec=$(ostree_file_path_to_object_path repo test2 baz/deeper/ohyeahx) +assert_file_has_mode ${objpath_exec} 755 +echo "ok bare-user committed modes" + +rm test2-checkout -rf +$OSTREE checkout -U -H test2 test2-checkout +cd test2-checkout +assert_file_has_mode baz/cow ${default_mode} +assert_file_has_mode baz/cowro 600 +assert_file_has_mode baz/deeper/ohyeahx 755 +echo "ok bare-user checkout modes" + +rm test2-checkout -rf +$OSTREE checkout -U -H test2 test2-checkout +touch test2-checkout/unwritable +chmod 0400 test2-checkout/unwritable +$OSTREE commit -b test2-unwritable --tree=dir=test2-checkout +chmod 0600 test2-checkout/unwritable +rm test2-checkout -rf +$OSTREE checkout -U -H test2-unwritable test2-checkout +cd test2-checkout +assert_file_has_mode unwritable 400 +echo "ok bare-user unwritable" From 81e1f7761fcab2e0e99f86e9c7751186f0f6731c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 13:12:08 -0400 Subject: [PATCH 57/86] tests: Add a test for bare-user-only failing to commit suid content We didn't have coverage of this before, and adding the test infrastructure will help ensure we have coverage for more changes here. Closes: #913 Approved by: alexlarsson --- tests/test-basic-user-only.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index a3e63aac..7184fe94 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -22,4 +22,28 @@ set -euo pipefail . $(dirname $0)/libtest.sh setup_test_repository "bare-user-only" +extra_basic_tests=1 . $(dirname $0)/basic-test.sh + +# Reset things so we don't inherit a lot of state from earlier tests +cd ${test_tmpdir} +rm repo files -rf +ostree_repo_init repo init --mode=bare-user-only + +# Init an archive repo where we'll store content that can't go into bare-user +cd ${test_tmpdir} +rm repo-input -rf +ostree_repo_init repo-input init --mode=archive +cd ${test_tmpdir} +cat > statoverride.txt < files/some-setuid +chmod 0644 files/some-setuid +$CMD_PREFIX ostree --repo=repo-input commit -b content-with-suid --statoverride=statoverride.txt --tree=dir=files +if $CMD_PREFIX ostree pull-local --repo=repo repo-input 2>err.txt; then + assert_not_reached "copying suid file into bare-user worked?" +fi +assert_file_has_content err.txt "Invalid mode.*with bits 040.*in bare-user-only" +echo "ok failed to commit suid" From c81252c1e071bf3c536a8dba5d2eb5979c87653d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 13:18:04 -0400 Subject: [PATCH 58/86] repo/commit: Support group-writable files for bare-user-only These exist in the wild for flatpak, and aren't really a problem. The canonical permissions are still either `0755` or `0644`, we just support the additional writable bit for the group (i.e. extend the set to include `0775` and `0664`) now to avoid breaking some flatpak content. Closes: #913 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 4 ++-- tests/test-basic-user-only.sh | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index da0a5cdc..93d22f65 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -301,10 +301,10 @@ commit_loose_object_trusted (OstreeRepo *self, self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY && !object_is_symlink) { - guint32 invalid_modebits = (mode & ~S_IFMT) & ~0755; + guint32 invalid_modebits = (mode & ~S_IFMT) & ~0775; if (invalid_modebits > 0) return glnx_throw (error, "Invalid mode 0%04o with bits 0%04o in bare-user-only repository", - mode, invalid_modebits); + mode, invalid_modebits); if (fchmod (fd, mode) < 0) return glnx_throw_errno_prefix (error, "fchmod"); diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index 7184fe94..deca0e00 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -22,7 +22,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh setup_test_repository "bare-user-only" -extra_basic_tests=1 +extra_basic_tests=2 . $(dirname $0)/basic-test.sh # Reset things so we don't inherit a lot of state from earlier tests @@ -47,3 +47,15 @@ if $CMD_PREFIX ostree pull-local --repo=repo repo-input 2>err.txt; then fi assert_file_has_content err.txt "Invalid mode.*with bits 040.*in bare-user-only" echo "ok failed to commit suid" + +cd ${test_tmpdir} +rm repo-input -rf +ostree_repo_init repo-input init --mode=archive +rm files -rf && mkdir files +echo "a group writable file" > files/some-group-writable +chmod 0664 files/some-group-writable +$CMD_PREFIX ostree --repo=repo-input commit -b content-with-group-writable --tree=dir=files +$CMD_PREFIX ostree pull-local --repo=repo repo-input +$CMD_PREFIX ostree --repo=repo checkout -U -H content-with-group-writable groupwritable-co +assert_file_has_mode groupwritable-co/some-group-writable 664 +echo "ok supported group writable" From 2a3f17c7aa8e5dc328ee9169dfa51d9cb579ba3a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 14 Dec 2015 10:58:53 +0100 Subject: [PATCH 59/86] repo: After renaming in all loose objects, ensure metadata is stable When a transaction is finished and we have moved all the staged loose objects into the repo we fsync all the object directory, to ensure the filenames are stable before we update the refs files to point to the new commits. With out this an unclean shutdown after the transaction is finished could result in a refs file that points to an incomplete commit. https://bugzilla.gnome.org/show_bug.cgi?id=759442 Closes: #918 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 36 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 93d22f65..be92a627 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1202,7 +1202,9 @@ rename_pending_loose_objects (OstreeRepo *self, while (TRUE) { struct dirent *dent; + gboolean renamed_some_object = FALSE; g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, }; + char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) return FALSE; @@ -1220,21 +1222,20 @@ rename_pending_loose_objects (OstreeRepo *self, &child_dfd_iter, error)) return FALSE; + loose_objpath[0] = dent->d_name[0]; + loose_objpath[1] = dent->d_name[1]; + loose_objpath[2] = '/'; + /* Iterate over inner checksum dir */ while (TRUE) { struct dirent *child_dent; - char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; if (!glnx_dirfd_iterator_next_dent (&child_dfd_iter, &child_dent, cancellable, error)) return FALSE; if (child_dent == NULL) break; - loose_objpath[0] = dent->d_name[0]; - loose_objpath[1] = dent->d_name[1]; - loose_objpath[2] = '/'; - g_strlcpy (loose_objpath + 3, child_dent->d_name, sizeof (loose_objpath)-3); if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_objpath, @@ -1244,9 +1245,34 @@ rename_pending_loose_objects (OstreeRepo *self, if (G_UNLIKELY (renameat (child_dfd_iter.fd, loose_objpath + 3, self->objects_dir_fd, loose_objpath) < 0)) return glnx_throw_errno (error); + + renamed_some_object = TRUE; + } + + if (renamed_some_object) + { + /* Ensure that in the case of a power cut all the directory metadata that + we want has reached the disk. In particular, we want this before we + update the refs to point to these objects. */ + glnx_fd_close int target_dir_fd = -1; + + loose_objpath[2] = 0; + + if (!glnx_opendirat (self->objects_dir_fd, + loose_objpath, FALSE, + &target_dir_fd, + error)) + return FALSE; + + if (fsync (target_dir_fd) == -1) + return glnx_throw_errno_prefix (error, "fsync"); } } + /* In case we created any loose object subdirs, make sure they are on disk */ + if (fsync (self->objects_dir_fd) == -1) + return glnx_throw_errno_prefix (error, "fsync"); + if (!glnx_shutil_rm_rf_at (self->tmp_dir_fd, self->commit_stagedir_name, cancellable, error)) return FALSE; From a98faa911343378781114d458ba3936c282b90ec Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 8 Jun 2017 13:47:20 -0400 Subject: [PATCH 60/86] ci: Update to match current rpm-ostree This copies the `ci/` directory from rpm-ostree, with much the same rationale; among other things we don't want to depend on the Docker hub. The specific reason I'm doing this is that I want to add a CentOS7 build, but that means we can't use `projectatomic/ostree-tester`, and at that point we might as well unwind it all. Closes: #917 Approved by: jlebon --- .papr.Dockerfile | 27 ------ .papr.yml | 97 +++----------------- ci/build-check.sh | 20 ++++ ci/build.sh | 15 +++ {tests => ci}/ci-commitmessage-submodules.sh | 0 ci/libbuild.sh | 23 +++++ 6 files changed, 72 insertions(+), 110 deletions(-) delete mode 100644 .papr.Dockerfile create mode 100755 ci/build-check.sh create mode 100755 ci/build.sh rename {tests => ci}/ci-commitmessage-submodules.sh (100%) create mode 100644 ci/libbuild.sh diff --git a/.papr.Dockerfile b/.papr.Dockerfile deleted file mode 100644 index 0081998c..00000000 --- a/.papr.Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM fedora:25 - -RUN dnf install -y \ - gcc \ - git \ - sudo \ - which \ - attr \ - fuse \ - gjs \ - parallel \ - coccinelle \ - clang \ - libubsan \ - libasan \ - libtsan \ - PyYAML \ - gnome-desktop-testing \ - redhat-rpm-config \ - elfutils \ - 'dnf-command(builddep)' \ - && dnf builddep -y \ - ostree \ - && dnf clean all - -# create an unprivileged user for testing -RUN adduser testuser diff --git a/.papr.yml b/.papr.yml index cc02f75e..67a01683 100644 --- a/.papr.yml +++ b/.papr.yml @@ -4,102 +4,51 @@ branches: - try required: true -context: f25-sanitizer +context: f25-primary container: - image: projectatomic/ostree-tester - + image: registry.fedoraproject.org/fedora:25 packages: - - libasan - git - - coccinelle env: CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc # TODO when we're doing leak checks: G_SLICE: "always-malloc" -build: - config-opts: > - --prefix=/usr - --libdir=/usr/lib64 - --enable-installed-tests=exclusive - --enable-gtk-doc - -# The g-d-t-r timeout is for test-pull-many.sh; if tweaking this, -# also be sure to change the other cases below tests: - - make syntax-check - - ./tests/ci-commitmessage-submodules.sh - - make check - - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' + - ci/ci-commitmessage-submodules.sh + - ci/build-check.sh timeout: 30m artifacts: - - test-suite.log - ---- - -inherit: true -required: true - -context: f25-clang - -env: - CC: 'clang' - CFLAGS: '-Werror=unused-variable' - -tests: -artifacts: - - + - test-suite.log --- inherit: true context: f25-rust - packages: - cargo - -build: - config-opts: > - --prefix=/usr - --libdir=/usr/lib64 - --enable-gtk-doc - --enable-rust - env: - CC: 'gcc' + CONFIGOPTS: '--enable-rust' tests: + - ci/build.sh - make check TESTS=tests/test-rollsum -artifacts: - - test-suite.log --- inherit: true context: f25-experimental-api - -build: - config-opts: > - --prefix=/usr - --libdir=/usr/lib64 - --enable-gtk-doc - --enable-experimental-api - env: - CC: 'gcc' + CONFIGOPTS: '--enable-experimental-api' tests: - - make check - - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' + - ci/build-check.sh -artifacts: - - test-suite.log --- inherit: true @@ -111,21 +60,11 @@ packages: - pkgconfig(libcurl) - pkgconfig(openssl) -build: - config-opts: > - --prefix=/usr - --libdir=/usr/lib64 - --enable-installed-tests=exclusive - --enable-gtk-doc - --with-curl - --with-openssl +env: + CONFIGOPTS: "--with-curl --with-openssl" tests: - - make check - - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' - -artifacts: - - test-suite.log + - ci/build-check.sh --- @@ -143,24 +82,16 @@ cluster: - name: vmcheck distro: fedora/25/atomic container: - image: projectatomic/ostree-tester - -build: - config-opts: > - --prefix=/usr - --libdir=/usr/lib64 - --enable-gtk-doc + image: registry.fedoraproject.org/fedora:25 # Copy the build from the container to the host; ideally down the line # this is installing an RPM via https://github.com/jlebon/redhat-ci/issues/10 tests: + - ci/build.sh - make install DESTDIR=$(pwd)/insttree - rsync -rl -e 'ssh -o User=root' . vmcheck:ostree/ - ssh root@vmcheck './ostree/tests/installed/fah-prep.sh && ostree admin unlock && rsync -rlv ./ostree/insttree/usr/ /usr/ && ./ostree/tests/installed/run.sh' -artifacts: - - test-suite.log - --- inherit: false diff --git a/ci/build-check.sh b/ci/build-check.sh new file mode 100755 index 00000000..6123440e --- /dev/null +++ b/ci/build-check.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash +# Install build dependencies, run unit tests and installed tests. + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libbuild.sh +${dn}/build.sh +make check +make syntax-check # TODO: do syntax-check under check +# And now run the installed tests +make install +gnome-desktop-testing-runner -p 0 ostree + +git clean -dfx && git submodule foreach git clean -dfx +# And now a clang build to find unused variables; perhaps +# in the future these could parallelize +export CC=clang +export CFLAGS='-Werror=unused-variable' +build diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 00000000..d5bacd38 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/bash +# Install build dependencies, run unit tests and installed tests. + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libbuild.sh + +install_builddeps ostree + +dnf install -y sudo which attr fuse gjs parallel coccinelle clang \ + libubsan libasan libtsan PyYAML gnome-desktop-testing redhat-rpm-config \ + elfutils + +build --enable-gtk-doc --enable-installed-tests=exclusive ${CONFIGOPTS:-} diff --git a/tests/ci-commitmessage-submodules.sh b/ci/ci-commitmessage-submodules.sh similarity index 100% rename from tests/ci-commitmessage-submodules.sh rename to ci/ci-commitmessage-submodules.sh diff --git a/ci/libbuild.sh b/ci/libbuild.sh new file mode 100644 index 00000000..b061a486 --- /dev/null +++ b/ci/libbuild.sh @@ -0,0 +1,23 @@ +#!/usr/bin/bash + +make() { + /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@" +} + +build() { + env NOCONFIGURE=1 ./autogen.sh + ./configure --prefix=/usr --libdir=/usr/lib64 "$@" + make V=1 +} + +install_builddeps() { + pkg=$1 + dnf -y install dnf-plugins-core + dnf install -y @buildsys-build + dnf install -y 'dnf-command(builddep)' + + # builddeps+runtime deps + dnf builddep -y $pkg + dnf install -y $pkg + rpm -e $pkg +} From e0e07e2a48ad61e497ee812d97db3cb1bb6e0d70 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 9 Jun 2017 10:57:40 -0400 Subject: [PATCH 61/86] ci: Add CentOS 7 build For similar reasons as https://github.com/projectatomic/rpm-ostree/pull/824 Closes: #919 Approved by: jlebon --- .papr.yml | 16 ++++++++++++++++ ci/build-check.sh | 18 +++++++++++------- ci/build.sh | 14 +++++++++----- ci/libbuild.sh | 45 ++++++++++++++++++++++++++++++++++++++------- 4 files changed, 74 insertions(+), 19 deletions(-) diff --git a/.papr.yml b/.papr.yml index 67a01683..74c16900 100644 --- a/.papr.yml +++ b/.papr.yml @@ -26,9 +26,25 @@ artifacts: - test-suite.log --- +context: c7-build inherit: true +required: true + +container: + image: registry.centos.org/centos/centos:7 + +env: + CFLAGS: '' + +tests: + - ci/build-check.sh + +--- context: f25-rust +inherit: true +container: + image: registry.fedoraproject.org/fedora:25 packages: - cargo env: diff --git a/ci/build-check.sh b/ci/build-check.sh index 6123440e..d15032e7 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -10,11 +10,15 @@ make check make syntax-check # TODO: do syntax-check under check # And now run the installed tests make install -gnome-desktop-testing-runner -p 0 ostree +if test -x /usr/bin/gnome-desktop-testing-runner; then + gnome-desktop-testing-runner -p 0 ostree +fi -git clean -dfx && git submodule foreach git clean -dfx -# And now a clang build to find unused variables; perhaps -# in the future these could parallelize -export CC=clang -export CFLAGS='-Werror=unused-variable' -build +if test -x /usr/bin/clang; then + git clean -dfx && git submodule foreach git clean -dfx + # And now a clang build to find unused variables; perhaps + # in the future these could parallelize + export CC=clang + export CFLAGS='-Werror=unused-variable' + build +fi diff --git a/ci/build.sh b/ci/build.sh index d5bacd38..eefb7c3c 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -6,10 +6,14 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libbuild.sh -install_builddeps ostree - -dnf install -y sudo which attr fuse gjs parallel coccinelle clang \ - libubsan libasan libtsan PyYAML gnome-desktop-testing redhat-rpm-config \ +pkg_install_builddeps ostree +pkg_install sudo which attr fuse \ + libubsan libasan libtsan PyYAML redhat-rpm-config \ elfutils +pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang -build --enable-gtk-doc --enable-installed-tests=exclusive ${CONFIGOPTS:-} +DETECTED_CONFIGOPTS= +if test -x /usr/bin/gnome-desktop-testing-runner; then + DETECTED_CONFIGOPTS="${DETECTED_CONFIGOPTS} --enable-installed-tests=exclusive" +fi +build --enable-gtk-doc ${DETECTED_CONFIGOPTS} ${CONFIGOPTS:-} diff --git a/ci/libbuild.sh b/ci/libbuild.sh index b061a486..fef9d3ae 100644 --- a/ci/libbuild.sh +++ b/ci/libbuild.sh @@ -10,14 +10,45 @@ build() { make V=1 } -install_builddeps() { - pkg=$1 - dnf -y install dnf-plugins-core - dnf install -y @buildsys-build - dnf install -y 'dnf-command(builddep)' +pkg_install() { + yum -y install "$@" +} +pkg_install_if_os() { + os=$1 + shift + (. /etc/os-release; + if test "${os}" = "${ID}"; then + pkg_install "$@" + else + echo "Skipping installation on OS ${ID}: $@" + fi + ) +} + +pkg_builddep() { + # This is sadly the only case where it's a different command + if test -x /usr/bin/dnf; then + dnf builddep -y "$@" + else + yum-builddep -y "$@" + fi +} + +pkg_install_builddeps() { + pkg=$1 + if test -x /usr/bin/dnf; then + yum -y install dnf-plugins-core + yum install -y 'dnf-command(builddep)' + # Base buildroot + pkg_install @buildsys-build + else + yum -y install yum-utils + # Base buildroot, copied from the mock config sadly + yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz + fi # builddeps+runtime deps - dnf builddep -y $pkg - dnf install -y $pkg + pkg_builddep $pkg + pkg_install $pkg rpm -e $pkg } From 5de201df2601cc51603b7270a0c7341b748fd29c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 10:05:02 -0400 Subject: [PATCH 62/86] repo: Fix leak of superblock fds when generating summary Related: https://github.com/ostreedev/ostree/issues/920 Closes: #921 Approved by: alexlarsson --- src/libostree/ostree-repo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 0d20c25b..86d41bbc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4549,7 +4549,6 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_autoptr(GInputStream) in_stream = g_unix_input_stream_new (superblock_file_fd, FALSE); if (!in_stream) return FALSE; - superblock_file_fd = -1; /* Transfer ownership */ g_autofree guchar *csum = NULL; if (!ot_gio_checksum_stream (in_stream, From aed8a6b09aecd5b71536148bae83bb29661c209c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 13:42:15 -0400 Subject: [PATCH 63/86] lib/commit: Port final object writing function to new code style I noticed my previous patches incorrectly started doing `return glnx_throw*` inside a `goto out;` function. Fix this by porting forward consistently to new style. We just do the error prefixing in the caller. Closes: #914 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 76 +++++++++--------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index be92a627..27d080ad 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -193,8 +193,6 @@ commit_loose_object_trusted (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - /* We may be writing as root to a non-root-owned repository; if so, * automatically inherit the non-root ownership. */ @@ -204,19 +202,13 @@ commit_loose_object_trusted (OstreeRepo *self, if (fd != -1) { if (fchown (fd, self->target_owner_uid, self->target_owner_gid) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fchown"); } else if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename, self->target_owner_uid, self->target_owner_gid, AT_SYMLINK_NOFOLLOW) == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fchownat"); } /* Special handling for symlinks in bare repositories */ @@ -235,48 +227,31 @@ commit_loose_object_trusted (OstreeRepo *self, if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename, uid, gid, AT_SYMLINK_NOFOLLOW) == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fchownat"); if (xattrs != NULL) { ot_security_smack_reset_dfd_name (self->tmp_dir_fd, temp_filename); if (!glnx_dfd_name_set_all_xattrs (self->tmp_dir_fd, temp_filename, xattrs, cancellable, error)) - goto out; + return FALSE; } } else { - int res; - if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE) { - do - res = fchown (fd, uid, gid); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + if (TEMP_FAILURE_RETRY (fchown (fd, uid, gid)) < 0) + return glnx_throw_errno_prefix (error, "fchown"); - do - res = fchmod (fd, mode); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + if (TEMP_FAILURE_RETRY (fchmod (fd, mode)) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); if (xattrs) { ot_security_smack_reset_fd (fd); if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error)) - goto out; + return FALSE; } } @@ -317,14 +292,8 @@ commit_loose_object_trusted (OstreeRepo *self, * set the modification time to OSTREE_TIMESTAMP. */ const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; - do - res = futimens (fd, times); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + if (TEMP_FAILURE_RETRY (futimens (fd, times)) < 0) + return glnx_throw_errno_prefix (error, "futimens"); } /* Ensure that in case of a power cut, these files have the data we @@ -333,23 +302,16 @@ commit_loose_object_trusted (OstreeRepo *self, if (!self->in_transaction && !self->disable_fsync) { if (fsync (fd) == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fsync"); } } if (!_ostree_repo_commit_loose_final (self, checksum, objtype, self->tmp_dir_fd, fd, temp_filename, cancellable, error)) - goto out; - - ret = TRUE; - out: - if (G_UNLIKELY (error && *error)) - g_prefix_error (error, "Writing object %s.%s: ", checksum, ostree_object_type_to_string (objtype)); - return ret; + return FALSE; + + return TRUE; } typedef struct @@ -551,7 +513,10 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, FALSE, uid, gid, mode, xattrs, state->fd, cancellable, error)) - goto out; + { + g_prefix_error (error, "Writing object %s.%s: ", checksum, ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); + goto out; + } } ret = TRUE; @@ -798,7 +763,8 @@ write_content_object (OstreeRepo *self, uid, gid, mode, xattrs, temp_fd, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Writing object %s.%s: ", actual_checksum, + ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE)); /* Clear the unlinker path, it was consumed */ tmp_unlinker.path = NULL; From 18ae8e5267f35394faa41cdeadd5125962ba9417 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 13:56:47 -0400 Subject: [PATCH 64/86] lib/commit: Drop some conditionals/clarify code in content path Both callers of `commit_loose_object_trusted()` were passing `OSTREE_OBJECT_TYPE_FILE`, so drop that parameter. This in turn allows us to drop lots of checking of that inside the function. Add a doc comment, and rename to `commit_loose_content_object()` for clarity. Closes: #914 Approved by: alexlarsson --- src/libostree/ostree-repo-commit.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 27d080ad..45b5bc6e 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -179,10 +179,13 @@ _ostree_repo_commit_loose_final (OstreeRepo *self, return TRUE; } +/* Given either a file or symlink, apply the final metadata to it depending on + * the repository mode. Note that @checksum is assumed to have been validated by + * the caller. + */ static gboolean -commit_loose_object_trusted (OstreeRepo *self, +commit_loose_content_object (OstreeRepo *self, const char *checksum, - OstreeObjectType objtype, const char *temp_filename, gboolean object_is_symlink, guint32 uid, @@ -239,7 +242,7 @@ commit_loose_object_trusted (OstreeRepo *self, } else { - if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE) + if (self->mode == OSTREE_REPO_MODE_BARE) { if (TEMP_FAILURE_RETRY (fchown (fd, uid, gid)) < 0) return glnx_throw_errno_prefix (error, "fchown"); @@ -254,9 +257,7 @@ commit_loose_object_trusted (OstreeRepo *self, return FALSE; } } - - if (objtype == OSTREE_OBJECT_TYPE_FILE && - self->mode == OSTREE_REPO_MODE_BARE_USER) + else if (self->mode == OSTREE_REPO_MODE_BARE_USER) { if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error)) return FALSE; @@ -272,8 +273,7 @@ commit_loose_object_trusted (OstreeRepo *self, return glnx_throw_errno_prefix (error, "fchmod"); } } - else if (objtype == OSTREE_OBJECT_TYPE_FILE && - self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY + else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY && !object_is_symlink) { guint32 invalid_modebits = (mode & ~S_IFMT) & ~0775; @@ -285,7 +285,7 @@ commit_loose_object_trusted (OstreeRepo *self, return glnx_throw_errno_prefix (error, "fchmod"); } - if (objtype == OSTREE_OBJECT_TYPE_FILE && _ostree_repo_mode_is_bare (self->mode)) + if (_ostree_repo_mode_is_bare (self->mode)) { /* To satisfy tools such as guile which compare mtimes * to determine whether or not source files need to be compiled, @@ -306,7 +306,7 @@ commit_loose_object_trusted (OstreeRepo *self, } } - if (!_ostree_repo_commit_loose_final (self, checksum, objtype, + if (!_ostree_repo_commit_loose_final (self, checksum, OSTREE_OBJECT_TYPE_FILE, self->tmp_dir_fd, fd, temp_filename, cancellable, error)) return FALSE; @@ -508,7 +508,7 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo *self, if (state->fd != -1) { - if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE, + if (!commit_loose_content_object (self, checksum, state->temp_filename, FALSE, uid, gid, mode, xattrs, state->fd, @@ -756,8 +756,7 @@ write_content_object (OstreeRepo *self, const guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); - if (!commit_loose_object_trusted (self, actual_checksum, - OSTREE_OBJECT_TYPE_FILE, + if (!commit_loose_content_object (self, actual_checksum, temp_filename, object_is_symlink, uid, gid, mode, From 8edb5161dbf9a94734baeb0332b367fe66aafbe4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Jun 2017 14:21:59 -0400 Subject: [PATCH 65/86] lib/checkout: Ignore world-writable dirs for bare-user-only checkout See https://github.com/ostreedev/ostree/pull/909 for more information on the rationale. Basically there's no reason for flatpak (which uses `bare-user-only`) to have world-writable dirs. Particularly with the presence of the system helper. An approach I considered instead was to parse and validate directory metadata objects at commit time. We still may do that in addition; for file objects we *had* to do it that way because the actual files would be laid down suid. But directories live only as inert `.dirmeta` objects until we do a checkout (i.e. `mkdir()`), so we can solve the problem at checkout time. Closes: #914 Approved by: alexlarsson --- src/libostree/ostree-repo-checkout.c | 14 ++++++++++++-- tests/test-basic-user-only.sh | 14 +++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index bb7c1771..2b259464 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -749,8 +749,18 @@ checkout_tree_at_recurse (OstreeRepo *self, */ if (!did_exist) { - if (TEMP_FAILURE_RETRY (fchmod (destination_dfd, mode)) < 0) - return glnx_throw_errno (error); + guint32 canonical_mode; + /* Silently ignore world-writable directories (plus sticky, suid bits, + * etc.) when doing a checkout for bare-user-only repos. This is related + * to the logic in ostree-repo-commit.c for files. + * See also: https://github.com/ostreedev/ostree/pull/909 i.e. 0c4b3a2b6da950fd78e63f9afec602f6188f1ab0 + */ + if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + canonical_mode = (mode & 0775) | S_IFDIR; + else + canonical_mode = mode; + if (TEMP_FAILURE_RETRY (fchmod (destination_dfd, canonical_mode)) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); } if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index deca0e00..29fbbdd3 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -22,7 +22,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh setup_test_repository "bare-user-only" -extra_basic_tests=2 +extra_basic_tests=3 . $(dirname $0)/basic-test.sh # Reset things so we don't inherit a lot of state from earlier tests @@ -59,3 +59,15 @@ $CMD_PREFIX ostree pull-local --repo=repo repo-input $CMD_PREFIX ostree --repo=repo checkout -U -H content-with-group-writable groupwritable-co assert_file_has_mode groupwritable-co/some-group-writable 664 echo "ok supported group writable" + +cd ${test_tmpdir} +rm repo-input -rf +ostree_repo_init repo-input init --mode=archive +rm files -rf && mkdir files +mkdir files/worldwritable-dir +chmod a+w files/worldwritable-dir +$CMD_PREFIX ostree --repo=repo-input commit -b content-with-dir-world-writable --tree=dir=files +$CMD_PREFIX ostree pull-local --repo=repo repo-input +$CMD_PREFIX ostree --repo=repo checkout -U -H content-with-dir-world-writable dir-co +assert_file_has_mode dir-co/worldwritable-dir 775 +echo "ok didn't make world-writable dir" From 848b7c020121fbda8d1028bf5e0e9adf8c591b0c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 15:36:16 -0400 Subject: [PATCH 66/86] lib/repo: Refactor object copy import function This came up in: https://github.com/ostreedev/ostree/pull/881 Basically doing streaming for metadata is dumb. Split up the metadata/content paths so we pass metadata around as `GVariant`. This drops the last internal caller of `ostree_repo_write_metadata_stream_trusted()` which was the dumb function mentioned. Closes: #923 Approved by: jlebon --- src/libostree/ostree-repo.c | 153 +++++++++++++++++------------------- 1 file changed, 72 insertions(+), 81 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 86d41bbc..bf126a91 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3049,77 +3049,6 @@ copy_detached_metadata (OstreeRepo *self, return TRUE; } -static gboolean -import_one_object_copy (OstreeRepo *self, - OstreeRepo *source, - const char *checksum, - OstreeObjectType objtype, - gboolean trusted, - GCancellable *cancellable, - GError **error) -{ - guint64 length; - g_autoptr(GInputStream) object_stream = NULL; - - if (!ostree_repo_load_object_stream (source, objtype, checksum, - &object_stream, &length, - cancellable, error)) - return FALSE; - - if (objtype == OSTREE_OBJECT_TYPE_FILE) - { - if (trusted) - { - if (!ostree_repo_write_content_trusted (self, checksum, - object_stream, length, - cancellable, error)) - return FALSE; - } - else - { - g_autofree guchar *real_csum = NULL; - if (!ostree_repo_write_content (self, checksum, - object_stream, length, - &real_csum, - cancellable, error)) - return FALSE; - } - } - else - { - if (objtype == OSTREE_OBJECT_TYPE_COMMIT) - { - if (!copy_detached_metadata (self, source, checksum, cancellable, error)) - return FALSE; - } - - if (trusted) - { - if (!ostree_repo_write_metadata_stream_trusted (self, objtype, - checksum, object_stream, length, - cancellable, error)) - return FALSE; - } - else - { - g_autofree guchar *real_csum = NULL; - g_autoptr(GVariant) variant = NULL; - - if (!ostree_repo_load_variant (source, objtype, checksum, - &variant, error)) - return FALSE; - - if (!ostree_repo_write_metadata (self, objtype, - checksum, variant, - &real_csum, - cancellable, error)) - return FALSE; - } - } - - return TRUE; -} - static gboolean import_one_object_link (OstreeRepo *self, OstreeRepo *source, @@ -3216,29 +3145,91 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean hardlink_was_supported = FALSE; - if (trusted && /* Don't hardlink into untrusted remotes */ self->mode == source->mode) { + gboolean hardlink_was_supported = FALSE; + if (!import_one_object_link (self, source, checksum, objtype, &hardlink_was_supported, cancellable, error)) return FALSE; + + /* If we hardlinked, we're done! */ + if (hardlink_was_supported) + return TRUE; } - if (!hardlink_was_supported) - { - gboolean has_object; + /* The copy path */ - if (!ostree_repo_has_object (self, objtype, checksum, &has_object, - cancellable, error)) + /* First, do we have the object already? */ + gboolean has_object; + if (!ostree_repo_has_object (self, objtype, checksum, &has_object, + cancellable, error)) + return FALSE; + /* If we have it, we're done */ + if (has_object) + return TRUE; + + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + { + /* Metadata object */ + g_autoptr(GVariant) variant = NULL; + + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + { + /* FIXME - cleanup detached metadata if copy below fails */ + if (!copy_detached_metadata (self, source, checksum, cancellable, error)) + return FALSE; + } + + if (!ostree_repo_load_variant (source, objtype, checksum, + &variant, error)) return FALSE; - if (!has_object) + if (trusted) { - if (!import_one_object_copy (self, source, checksum, objtype, trusted, - cancellable, error)) + if (!ostree_repo_write_metadata_trusted (self, objtype, + checksum, variant, + cancellable, error)) + return FALSE; + } + else + { + g_autofree guchar *real_csum = NULL; + + if (!ostree_repo_write_metadata (self, objtype, + checksum, variant, + &real_csum, + cancellable, error)) + return FALSE; + } + } + else + { + /* Content object */ + guint64 length; + g_autoptr(GInputStream) object_stream = NULL; + + if (!ostree_repo_load_object_stream (source, objtype, checksum, + &object_stream, &length, + cancellable, error)) + return FALSE; + + if (trusted) + { + if (!ostree_repo_write_content_trusted (self, checksum, + object_stream, length, + cancellable, error)) + return FALSE; + } + else + { + g_autofree guchar *real_csum = NULL; + if (!ostree_repo_write_content (self, checksum, + object_stream, length, + &real_csum, + cancellable, error)) return FALSE; } } From 7159bed8e182781ccdc9a5913457cf6fda0bcfbe Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 8 Jun 2017 09:43:23 +0200 Subject: [PATCH 67/86] lib/repo: Always look in staging directory for objects Its often the case that we want to look at objects inside a commit, before the objects the transaction is finished. For instance: https://github.com/flatpak/flatpak/pull/837 Which tries to verify the file permissions before committing the transaction. And: https://github.com/flatpak/flatpak/commit/1e5ffa926a25acb655af7889b679b140bf44870b Which collects the storage size of the objects so that we can put the total download size in the commit metadata. I tried to find all the places where we did reads from the object directories, and in particular this fixes: - `ostree_repo_load_file()` for `bare` repos (`archive` was already working). - `ostree_repo_query_object_storage_size()` - Applying deltas that reference not-yet-commited objects Closes: #916 Approved by: cgwalters --- src/libostree/ostree-repo.c | 58 +++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index bf126a91..1da06217 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2501,19 +2501,33 @@ load_metadata_internal (OstreeRepo *self, return TRUE; } +/* Basically fstatat(), but also looks in both the committed and staging + * directories, and returns *out_dfd for where we found the object. + */ static gboolean -query_info_for_bare_content_object (OstreeRepo *self, - const char *loose_path_buf, - GFileInfo **out_info, - GCancellable *cancellable, - GError **error) +stat_bare_content_object (OstreeRepo *self, + const char *loose_path_buf, + int *out_dfd, + GFileInfo **out_info, + GCancellable *cancellable, + GError **error) { struct stat stbuf; + int res; + int dirfd; - if (TEMP_FAILURE_RETRY (fstatat (self->objects_dir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) + dirfd = self->objects_dir_fd; + res = TEMP_FAILURE_RETRY (fstatat (dirfd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)); + if (res < 0 && errno == ENOENT && self->commit_stagedir_fd != -1) + { + dirfd = self->commit_stagedir_fd; + res = TEMP_FAILURE_RETRY (fstatat (dirfd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)); + } + if (res < 0) { if (errno == ENOENT) { + *out_dfd = -1; *out_info = NULL; return TRUE; } @@ -2528,13 +2542,14 @@ query_info_for_bare_content_object (OstreeRepo *self, } else if (S_ISLNK (stbuf.st_mode)) { - if (!ot_readlinkat_gfile_info (self->objects_dir_fd, loose_path_buf, + if (!ot_readlinkat_gfile_info (dirfd, loose_path_buf, ret_info, cancellable, error)) return FALSE; } else return glnx_throw (error, "Not a regular file or symlink: %s", loose_path_buf); + *out_dfd = dirfd; ot_transfer_out_value (out_info, &ret_info); return TRUE; } @@ -2575,6 +2590,12 @@ _ostree_repo_read_bare_fd (OstreeRepo *self, if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, out_fd, error)) return FALSE; + if (*out_fd == -1 && self->commit_stagedir_fd != -1) + { + if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, out_fd, error)) + return FALSE; + } + if (*out_fd == -1) { if (self->parent_repo) @@ -2661,9 +2682,11 @@ ostree_repo_load_file (OstreeRepo *self, } else { - if (!query_info_for_bare_content_object (self, loose_path_buf, - &ret_file_info, - cancellable, error)) + int objdir_fd; /* referenced */ + if (!stat_bare_content_object (self, loose_path_buf, + &objdir_fd, + &ret_file_info, + cancellable, error)) return FALSE; if (ret_file_info) @@ -2681,7 +2704,7 @@ ostree_repo_load_file (OstreeRepo *self, * always do an open, then query the user.ostreemeta xattr for * more information. */ - fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) return glnx_throw_errno (error); @@ -2735,7 +2758,7 @@ ostree_repo_load_file (OstreeRepo *self, if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR && out_input) { - fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) return glnx_throw_errno (error); @@ -2759,7 +2782,7 @@ ostree_repo_load_file (OstreeRepo *self, { glnx_fd_close int fd = -1; - fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + fd = openat (objdir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) return glnx_throw_errno (error); @@ -2783,7 +2806,7 @@ ostree_repo_load_file (OstreeRepo *self, { if (self->disable_xattrs) ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); - else if (!glnx_dfd_name_get_all_xattrs (self->objects_dir_fd, loose_path_buf, + else if (!glnx_dfd_name_get_all_xattrs (objdir_fd, loose_path_buf, &ret_xattrs, cancellable, error)) return FALSE; @@ -3260,9 +3283,14 @@ ostree_repo_query_object_storage_size (OstreeRepo *self, { char loose_path[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (loose_path, sha256, objtype, self->mode); + int res; struct stat stbuf; - if (TEMP_FAILURE_RETRY (fstatat (self->objects_dir_fd, loose_path, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) + res = TEMP_FAILURE_RETRY (fstatat (self->objects_dir_fd, loose_path, &stbuf, AT_SYMLINK_NOFOLLOW)); + if (res < 0 && errno == ENOENT && self->commit_stagedir_fd != -1) + res = TEMP_FAILURE_RETRY (fstatat (self->commit_stagedir_fd, loose_path, &stbuf, AT_SYMLINK_NOFOLLOW)); + + if (res < 0) return glnx_throw_errno_prefix (error, "Querying object %s.%s", sha256, ostree_object_type_to_string (objtype)); *out_size = stbuf.st_size; From 21eec96bfdc553fc77f842776758c3dfe61e0185 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 7 Jun 2017 14:29:55 +0100 Subject: [PATCH 68/86] lib/pull: Fix construction of a refspec to use the correct separator This code looks like it was supposed to build a refspec, but it used a slash as a separator rather than a colon. The following code does recover by supporting prefix matching with slashes, but it seems like this was perhaps not the intention. Signed-off-by: Philip Withnall Closes: #912 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ed782cd5..7f6d0123 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3392,7 +3392,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autofree char *original_rev = NULL; if (pull_data->remote_name) - remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, ref); + remote_ref = g_strdup_printf ("%s:%s", pull_data->remote_name, ref); else remote_ref = g_strdup (ref); From 695771667c7736f3c69b74a6a3f344b36aff1e41 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 13:20:42 -0400 Subject: [PATCH 69/86] lib/repo: Skip import via hardlink if repo owners don't match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before this, if one had repos of matching mode but different owners, which could happen if one e.g. makes a `bare` non-root repo in `/ostree/deploy/$stateroot/var/tmp`, every time we tried to call `linkat()` we'd get `EPERM` and fall back to a copy. Fix this by saving the repo owner uid, and avoid trying to call `linkat()` if we know it's going to fail. Of course most commonly in this scenario we'll immediately fail trying to `chown` the files to `0`, but this is prep for a future patch to improve `bare-user` → `bare-user-only` imports where we'll be a bit more sophisticated. Closes: #922 Approved by: alexlarsson --- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 55602940..6cbf9ebe 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -116,6 +116,7 @@ struct OstreeRepo { GHashTable *updated_uncompressed_dirs; GHashTable *object_sizes; + uid_t owner_uid; uid_t target_owner_uid; gid_t target_owner_gid; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1da06217..e43d4dcd 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2092,6 +2092,7 @@ ostree_repo_open (OstreeRepo *self, if (fstat (self->objects_dir_fd, &stbuf) != 0) return glnx_throw_errno (error); + self->owner_uid = stbuf.st_uid; if (stbuf.st_uid != getuid () || stbuf.st_gid != getgid ()) { @@ -3168,8 +3169,14 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, GCancellable *cancellable, GError **error) { - if (trusted && /* Don't hardlink into untrusted remotes */ - self->mode == source->mode) + /* We try to import via hardlink. If the remote is explicitly not trusted + * (i.e.) their checksums may be incorrect, we skip that. Also, we require the + * repository modes to match, as well as the owner uid (since we need to be + * able to make hardlinks). + */ + if (trusted && + self->mode == source->mode && + self->owner_uid == source->owner_uid) { gboolean hardlink_was_supported = FALSE; From b614c65eabc416458dc7d27f8339f84091f68ad2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 13:38:52 -0400 Subject: [PATCH 70/86] lib/repo: Import metadata via hardlink even for distinct repo modes Our previous logic for import-via-hardlink only tried if the repo modes match, but we *can* hardlink metadata between e.g. `archive` and `bare-user` repos, and that's quite useful thing to do. Our documentation encourages converting to/from those repo modes locally for build systems. Closes: #922 Approved by: alexlarsson --- src/libostree/ostree-repo.c | 21 ++++++++++++++++++--- tests/basic-test.sh | 13 +++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e43d4dcd..33ca0ae8 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3143,6 +3143,23 @@ ostree_repo_import_object_from (OstreeRepo *self, checksum, TRUE, cancellable, error); } +static gboolean +import_via_hardlink_is_possible (OstreeRepo *src_repo, + OstreeRepo *dest_repo, + OstreeObjectType objtype) +{ + /* We need the ability to make hardlinks */ + if (src_repo->owner_uid != dest_repo->owner_uid) + return FALSE; + /* Equal modes are always compatible */ + if (src_repo->mode == dest_repo->mode) + return TRUE; + /* Metadata is identical between all modes */ + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + return TRUE; + return FALSE; +} + /** * ostree_repo_import_object_from_with_trust: * @self: Destination repo @@ -3174,9 +3191,7 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, * repository modes to match, as well as the owner uid (since we need to be * able to make hardlinks). */ - if (trusted && - self->mode == source->mode && - self->owner_uid == source->owner_uid) + if (trusted && import_via_hardlink_is_possible (source, self, objtype)) { gboolean hardlink_was_supported = FALSE; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index e3e442e9..f6aa72c2 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -254,9 +254,18 @@ echo "ok diff file changing type" cd ${test_tmpdir} mkdir repo2 -ostree_repo_init repo2 --mode=bare-user +# Use a different mode to test hardlinking metadata only +if grep -q 'mode=archive' repo/config || is_bare_user_only_repo repo; then + opposite_mode=bare-user +else + opposite_mode=archive +fi +ostree_repo_init repo2 --mode=$opposite_mode ${CMD_PREFIX} ostree --repo=repo2 pull-local repo -echo "ok pull-local" +test2_commitid=$(${CMD_PREFIX} ostree --repo=repo rev-parse test2) +test2_commit_relpath=/objects/${test2_commitid:0:2}/${test2_commitid:2}.commit +assert_files_hardlinked repo/${test2_commit_relpath} repo2/${test2_commit_relpath} +echo "ok pull-local (hardlinking metadata)" cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo2 checkout ${CHECKOUT_U_ARG} test2 test2-checkout-from-local-clone From 74e3581ed674746d03e8a0e0af332b10de26963f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 13:59:33 -0400 Subject: [PATCH 71/86] lib/repo: Support hardlink conversions from bare-user to bu-only Thinking about the problem of flatpak converting from `bare-user` to `bare-user-only` "in place" by creating a new repo and doing a `pull-local`, I realized that we can optimize this process by doing hardlinks for both metadata and regular files. The repo formats are *almost* compatible, the exception being symlinks. An earlier patch caused us to do hardlinks for metadata, this patch takes things to the next step and special cases this specific conversion. In this case we need to parse the source object to determine whether or not it's a symlink. Closes: #922 Approved by: alexlarsson --- src/libostree/ostree-repo.c | 46 +++++++++++++++++++++++++++++++++++ tests/libtest.sh | 30 +++++++++++++++++------ tests/test-basic-user-only.sh | 21 +++++++++++++++- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 33ca0ae8..49251284 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3073,6 +3073,20 @@ copy_detached_metadata (OstreeRepo *self, return TRUE; } +/* Special case between bare-user and bare-user-only, + * mostly for https://github.com/flatpak/flatpak/issues/845 + * see below for any more comments. + */ +static gboolean +import_is_bareuser_only_conversion (OstreeRepo *src_repo, + OstreeRepo *dest_repo, + OstreeObjectType objtype) +{ + return src_repo->mode == OSTREE_REPO_MODE_BARE_USER + && dest_repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY + && objtype == OSTREE_OBJECT_TYPE_FILE; +} + static gboolean import_one_object_link (OstreeRepo *self, OstreeRepo *source, @@ -3085,6 +3099,33 @@ import_one_object_link (OstreeRepo *self, char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (loose_path_buf, checksum, objtype, self->mode); + /* Hardlinking between bare-user → bare-user-only is only possible for regular + * files, *not* symlinks, which in bare-user are stored as regular files. At + * this point we need to parse the file to see the difference. + */ + if (import_is_bareuser_only_conversion (source, self, objtype)) + { + g_autoptr(GFileInfo) finfo = NULL; + + if (!ostree_repo_load_file (source, checksum, NULL, &finfo, NULL, + cancellable, error)) + return FALSE; + + switch (g_file_info_get_file_type (finfo)) + { + case G_FILE_TYPE_REGULAR: + /* This is OK, we'll drop through and try a hardlink */ + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + /* NOTE early return */ + *out_was_supported = FALSE; + return TRUE; + default: + g_assert_not_reached (); + break; + } + } + if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path_buf, cancellable, error)) return FALSE; @@ -3157,6 +3198,11 @@ import_via_hardlink_is_possible (OstreeRepo *src_repo, /* Metadata is identical between all modes */ if (OSTREE_OBJECT_TYPE_IS_META (objtype)) return TRUE; + /* And now a special case between bare-user and bare-user-only, + * mostly for https://github.com/flatpak/flatpak/issues/845 + */ + if (import_is_bareuser_only_conversion (src_repo, dest_repo, objtype)) + return TRUE; return FALSE; } diff --git a/tests/libtest.sh b/tests/libtest.sh index 1774a7b6..15802dfd 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -116,10 +116,16 @@ else fi fi +files_are_hardlinked() { + f1=$(stat -c %i $1) + f2=$(stat -c %i $2) + [ "$f1" == "$f2" ] +} + assert_files_hardlinked() { f1=$(stat -c %i $1) f2=$(stat -c %i $2) - if [ "$f1" != "$f2" ]; then + if ! files_are_hardlinked "$f1" "$f2"; then fatal "Files '$1' and '$2' are not hardlinked" fi } @@ -512,12 +518,22 @@ ostree_file_path_to_checksum() { $CMD_PREFIX ostree --repo=$repo ls -C $ref $path | awk '{ print $5 }' } +# Given a path to a file in a repo for a ref, print the (relative) path to its +# object +ostree_file_path_to_relative_object_path() { + repo=$1 + ref=$2 + path=$3 + checksum=$(ostree_file_path_to_checksum $repo $ref $path) + test -n "${checksum}" + echo objects/${checksum:0:2}/${checksum:2}.file +} + # Given a path to a file in a repo for a ref, print the path to its object ostree_file_path_to_object_path() { - repo=$1 - ref=$2 - path=$3 - checksum=$(ostree_file_path_to_checksum $repo $ref $path) - test -n "${checksum}" - echo ${repo}/objects/${checksum:0:2}/${checksum:2}.file + repo=$1 + ref=$2 + path=$3 + relpath=$(ostree_file_path_to_relative_object_path $repo $ref $path) + echo ${repo}/${relpath} } diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index 29fbbdd3..fb071fe4 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -22,7 +22,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh setup_test_repository "bare-user-only" -extra_basic_tests=3 +extra_basic_tests=4 . $(dirname $0)/basic-test.sh # Reset things so we don't inherit a lot of state from earlier tests @@ -71,3 +71,22 @@ $CMD_PREFIX ostree pull-local --repo=repo repo-input $CMD_PREFIX ostree --repo=repo checkout -U -H content-with-dir-world-writable dir-co assert_file_has_mode dir-co/worldwritable-dir 775 echo "ok didn't make world-writable dir" + +cd ${test_tmpdir} +rm repo-input -rf +rm repo -rf +ostree_repo_init repo init --mode=bare-user-only +ostree_repo_init repo-input init --mode=bare-user +rm files -rf && mkdir files +echo afile > files/afile +ln -s afile files/afile-link +$CMD_PREFIX ostree --repo=repo-input commit --canonical-permissions -b testtree --tree=dir=files +afile_relobjpath=$(ostree_file_path_to_relative_object_path repo-input testtree /afile) +afile_link_relobjpath=$(ostree_file_path_to_relative_object_path repo-input testtree /afile-link) +$CMD_PREFIX ostree pull-local --repo=repo repo-input +assert_files_hardlinked repo/${afile_relobjpath} repo-input/${afile_relobjpath} +if files_are_hardlinked repo/${afile_link_relobjpath} repo-input/${afile_link_relobjpath}; then + assert_not_reached "symlinks hardlinked across bare-user?" +fi +$OSTREE fsck -q +echo "ok hardlink pull from bare-user" From 6ed824bf00cda6943bdc221194d65de9382d79d8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Jun 2017 15:06:19 -0400 Subject: [PATCH 72/86] lib/pull: Add OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES This is an option which is intended mostly for flatpak; see: https://github.com/flatpak/flatpak/issues/845 We're adding an option for pulling into *all* repo modes that has an effect similar to the `bare-user-only` change from https://github.com/ostreedev/ostree/pull/909 This way one can pull content into e.g. a root-owned `bare` repository and ensure that there aren't any setuid or world-writable files. Closes: #926 Approved by: alexlarsson --- src/libostree/ostree-repo-pull.c | 124 ++++++++++++++++++++++++++--- src/libostree/ostree-repo.h | 4 +- src/ostree/ot-builtin-pull-local.c | 4 + src/ostree/ot-builtin-pull.c | 4 + tests/basic-test.sh | 28 ++++++- tests/pull-test.sh | 9 ++- 6 files changed, 160 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7f6d0123..2e93f90b 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -111,6 +111,7 @@ typedef struct { gboolean is_mirror; gboolean is_commit_only; gboolean is_untrusted; + gboolean is_bareuseronly_files; GPtrArray *dirs; @@ -556,6 +557,94 @@ pull_matches_subdir (OtPullData *pull_data, return FALSE; } +/* This bit mirrors similar code in commit_loose_content_object() for the + * bare-user-only mode. It's opt-in though for all pulls. + */ +static gboolean +validate_bareuseronly_mode (OtPullData *pull_data, + const char *checksum, + guint32 content_mode, + GError **error) +{ + if (!pull_data->is_bareuseronly_files) + return TRUE; + + if (S_ISREG (content_mode)) + { + const guint32 invalid_modebits = ((content_mode & ~S_IFMT) & ~0775); + if (invalid_modebits > 0) + return glnx_throw (error, "object %s.file: invalid mode 0%04o with bits 0%04o", + checksum, content_mode, invalid_modebits); + } + else if (S_ISLNK (content_mode)) + ; /* Nothing */ + else + g_assert_not_reached (); + + return TRUE; +} + +/* Import a single content object in the case where + * we have pull_data->remote_repo_local. + * + * One important special case here is handling the + * OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES flag. + */ +static gboolean +import_one_local_content_object (OtPullData *pull_data, + const char *checksum, + GCancellable *cancellable, + GError **error) +{ + g_assert (pull_data->remote_repo_local); + + const gboolean trusted = !pull_data->is_untrusted; + if (trusted && !pull_data->is_bareuseronly_files) + { + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + OSTREE_OBJECT_TYPE_FILE, checksum, + trusted, + cancellable, error)) + return FALSE; + } + else + { + /* In this case we either need to validate the checksum + * or the file mode. + */ + g_autoptr(GInputStream) content_input = NULL; + g_autoptr(GFileInfo) content_finfo = NULL; + g_autoptr(GVariant) content_xattrs = NULL; + + if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum, + &content_input, &content_finfo, &content_xattrs, + cancellable, error)) + return FALSE; + + if (!validate_bareuseronly_mode (pull_data, checksum, + g_file_info_get_attribute_uint32 (content_finfo, "unix::mode"), + error)) + return FALSE; + + /* Now that we've potentially validated it, convert to object stream */ + guint64 length; + g_autoptr(GInputStream) object_stream = NULL; + if (!ostree_raw_file_to_content_stream (content_input, content_finfo, + content_xattrs, &object_stream, + &length, cancellable, error)) + return FALSE; + + g_autofree guchar *real_csum = NULL; + if (!ostree_repo_write_content (pull_data->repo, checksum, + object_stream, length, + &real_csum, + cancellable, error)) + return FALSE; + } + + return TRUE; +} + static gboolean scan_dirtree_object (OtPullData *pull_data, const char *checksum, @@ -595,15 +684,19 @@ scan_dirtree_object (OtPullData *pull_data, &file_is_stored, cancellable, error)) return FALSE; - if (!file_is_stored && pull_data->remote_repo_local) + /* If we already have this object, move on to the next */ + if (file_is_stored) + continue; + + /* Is this a local repo? */ + if (pull_data->remote_repo_local) { - if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, - OSTREE_OBJECT_TYPE_FILE, file_checksum, !pull_data->is_untrusted, - cancellable, error)) + if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error)) return FALSE; } - else if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum)) + else if (!g_hash_table_lookup (pull_data->requested_content, file_checksum)) { + /* In this case we're doing HTTP pulls */ g_hash_table_add (pull_data->requested_content, file_checksum); enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); file_checksum = NULL; /* Transfer ownership */ @@ -2775,6 +2868,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; pull_data->is_untrusted = (flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) > 0; + pull_data->is_bareuseronly_files = (flags & OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES) > 0; pull_data->cancellable = cancellable ? g_object_ref (cancellable) : NULL; if (error) @@ -3042,11 +3136,21 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - /* For local pulls, default to disabling static deltas so that the - * exact object files are copied. - */ - if (pull_data->remote_repo_local && !pull_data->require_static_deltas) - pull_data->disable_static_deltas = TRUE; + if (pull_data->remote_repo_local) + { + /* For local pulls, default to disabling static deltas so that the + * exact object files are copied. + */ + if (!pull_data->require_static_deltas) + pull_data->disable_static_deltas = TRUE; + + } + else if (pull_data->is_bareuseronly_files) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't use bareuseronly-files with non-local origin repo"); + goto out; + } /* We can't use static deltas if pulling into an archive-z2 repo. */ if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 86bed09c..beacbd58 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1015,12 +1015,14 @@ gboolean ostree_repo_prune_from_reachable (OstreeRepo *self, * @OSTREE_REPO_PULL_FLAGS_MIRROR: Write out refs suitable for mirrors * @OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY: Fetch only the commit metadata * @OSTREE_REPO_PULL_FLAGS_UNTRUSTED: Don't trust local remote + * @OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES: Since 2017.7. Reject writes of content objects with modes outside of 0775. */ typedef enum { OSTREE_REPO_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR = (1 << 0), OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY = (1 << 1), - OSTREE_REPO_PULL_FLAGS_UNTRUSTED = (1 << 2) + OSTREE_REPO_PULL_FLAGS_UNTRUSTED = (1 << 2), + OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES = (1 << 3) } OstreeRepoPullFlags; _OSTREE_PUBLIC diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 28717b85..b19a4c6a 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -33,6 +33,7 @@ static char *opt_remote; static gboolean opt_disable_fsync; static gboolean opt_untrusted; +static gboolean opt_bareuseronly_files; static gboolean opt_require_static_deltas; static gboolean opt_gpg_verify; static gboolean opt_gpg_verify_summary; @@ -42,6 +43,7 @@ static GOptionEntry options[] = { { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust source", NULL }, + { "bareuseronly-files", 0, 0, G_OPTION_ARG_NONE, &opt_bareuseronly_files, "Reject regular files with mode outside of 0775 (world writable, suid, etc.)", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify, "GPG verify commits (must specify --remote)", NULL }, { "gpg-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify_summary, "GPG verify summary (must specify --remote)", NULL }, @@ -92,6 +94,8 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr if (opt_untrusted) pullflags |= OSTREE_REPO_PULL_FLAGS_UNTRUSTED; + if (opt_bareuseronly_files) + pullflags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index d88c5ee8..bd3ac115 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -34,6 +34,7 @@ static gboolean opt_dry_run; static gboolean opt_disable_static_deltas; static gboolean opt_require_static_deltas; static gboolean opt_untrusted; +static gboolean opt_bareuseronly_files; static char** opt_subpaths; static char** opt_http_headers; static char* opt_cache_dir; @@ -50,6 +51,7 @@ static GOptionEntry options[] = { { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, { "subpath", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_subpaths, "Only pull the provided subpath(s)", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust (local) sources", NULL }, + { "bareuseronly-files", 0, 0, G_OPTION_ARG_NONE, &opt_bareuseronly_files, "Reject regular files with mode outside of 0775 (world writable, suid, etc.)", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, @@ -167,6 +169,8 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_untrusted) pullflags |= OSTREE_REPO_PULL_FLAGS_UNTRUSTED; + if (opt_bareuseronly_files) + pullflags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; if (opt_dry_run && !opt_require_static_deltas) { diff --git a/tests/basic-test.sh b/tests/basic-test.sh index f6aa72c2..ba051e03 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..$((66 + ${extra_basic_tests:-0}))" +echo "1..$((68 + ${extra_basic_tests:-0}))" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' @@ -267,6 +267,32 @@ test2_commit_relpath=/objects/${test2_commitid:0:2}/${test2_commitid:2}.commit assert_files_hardlinked repo/${test2_commit_relpath} repo2/${test2_commit_relpath} echo "ok pull-local (hardlinking metadata)" +cd ${test_tmpdir} +rm repo2 -rf && mkdir repo2 +ostree_repo_init repo2 --mode=$opposite_mode +${CMD_PREFIX} ostree --repo=repo2 pull-local --bareuseronly-files repo test2 +${CMD_PREFIX} ostree --repo=repo2 fsck -q +echo "ok pull-local --bareuseronly-files" + +# This is mostly a copy of the suid test in test-basic-user-only.sh, +# but for the `pull --bareuseronly-files` case. +cd ${test_tmpdir} +rm repo-input -rf +ostree_repo_init repo-input init --mode=archive +cd ${test_tmpdir} +cat > statoverride.txt < files/some-setuid +chmod 0644 files/some-setuid +$CMD_PREFIX ostree --repo=repo-input commit -b content-with-suid --statoverride=statoverride.txt --tree=dir=files +if $CMD_PREFIX ostree pull-local --repo=repo --bareuseronly-files repo-input content-with-suid 2>err.txt; then + assert_not_reached "copying suid file with --bareuseronly-files worked?" +fi +assert_file_has_content err.txt 'object.*\.file: invalid mode.*with bits 040.*' +echo "ok pull-local (bareuseronly files)" + cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo2 checkout ${CHECKOUT_U_ARG} test2 test2-checkout-from-local-clone cd test2-checkout-from-local-clone diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 4a4ef069..f1ebca4c 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..20" +echo "1..21" # Try both syntaxes repo_init --no-gpg-verify @@ -79,6 +79,13 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo pull origin main ${CMD_PREFIX} ostree --repo=mirrorrepo fsck echo "ok pull (refuses deltas)" +if ${CMD_PREFIX} ostree --repo=mirrorrepo \ + pull origin main --bareuseronly-files 2>err.txt; then + assert_not_reached "--bareuseronly-files unexpectedly succeeded" +fi +assert_file_has_content err.txt 'bareuseronly-files with non-local' +echo "ok pull (refuses bareuseronly)" + cd ${test_tmpdir} rm mirrorrepo/refs/remotes/* -rf ${CMD_PREFIX} ostree --repo=mirrorrepo prune --refs-only From 0635fcbfd9d1efa298ffc08a4a8d346e740292fd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 13 Jun 2017 13:26:33 -0400 Subject: [PATCH 73/86] lib/checkout: Add bareuseronly_dirs option This is a continuation of https://github.com/ostreedev/ostree/pull/926 for directories instead of files. See: https://github.com/flatpak/flatpak/issues/845 This option suppresses mode bits outside of `0775` for directory checkouts. I think most people should start doing this by default, and use explicit overrides for e.g. `/tmp` if doing a recommit based on a checkout. Closes: #927 Approved by: alexlarsson --- src/libostree/ostree-repo-checkout.c | 6 +++--- src/libostree/ostree-repo.h | 3 ++- src/ostree/ot-builtin-checkout.c | 6 +++++- tests/basic-test.sh | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 2b259464..4b14dcdb 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -751,11 +751,11 @@ checkout_tree_at_recurse (OstreeRepo *self, { guint32 canonical_mode; /* Silently ignore world-writable directories (plus sticky, suid bits, - * etc.) when doing a checkout for bare-user-only repos. This is related - * to the logic in ostree-repo-commit.c for files. + * etc.) when doing a checkout for bare-user-only repos, or if requested explicitly. + * This is related to the logic in ostree-repo-commit.c for files. * See also: https://github.com/ostreedev/ostree/pull/909 i.e. 0c4b3a2b6da950fd78e63f9afec602f6188f1ab0 */ - if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY || options->bareuseronly_dirs) canonical_mode = (mode & 0775) | S_IFDIR; else canonical_mode = mode; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index beacbd58..ed73d4a2 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -768,7 +768,8 @@ typedef struct { gboolean process_whiteouts; gboolean no_copy_fallback; gboolean force_copy; /* Since: 2017.6 */ - gboolean unused_bools[6]; + gboolean bareuseronly_dirs; /* Since: 2017.7 */ + gboolean unused_bools[5]; const char *subpath; diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 8ffe5bb7..937c5b96 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -43,6 +43,7 @@ static char *opt_from_file; static gboolean opt_disable_fsync; static gboolean opt_require_hardlinks; static gboolean opt_force_copy; +static gboolean opt_bareuseronly_dirs; static gboolean parse_fsync_cb (const char *option_name, @@ -73,6 +74,7 @@ static GOptionEntry options[] = { { "fsync", 0, 0, G_OPTION_ARG_CALLBACK, parse_fsync_cb, "Specify how to invoke fsync()", "POLICY" }, { "require-hardlinks", 'H', 0, G_OPTION_ARG_NONE, &opt_require_hardlinks, "Do not fall back to full copies if hardlinking fails", NULL }, { "force-copy", 'C', 0, G_OPTION_ARG_NONE, &opt_force_copy, "Never hardlink (but may reflink if available)", NULL }, + { "bareuseronly-dirs", 'M', 0, G_OPTION_ARG_NONE, &opt_bareuseronly_dirs, "Suppress mode bits outside of 0775 for directories (suid, world writable, etc.)", NULL }, { NULL } }; @@ -91,7 +93,8 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add || opt_force_copy) + if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || + opt_union_add || opt_force_copy || opt_bareuseronly_dirs) { OstreeRepoCheckoutAtOptions options = { 0, }; @@ -119,6 +122,7 @@ process_one_checkout (OstreeRepo *repo, options.subpath = subpath; options.no_copy_fallback = opt_require_hardlinks; options.force_copy = opt_force_copy; + options.bareuseronly_dirs = opt_bareuseronly_dirs; if (!ostree_repo_checkout_at (repo, &options, AT_FDCWD, destination, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index ba051e03..d9b20938 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..$((68 + ${extra_basic_tests:-0}))" +echo "1..$((69 + ${extra_basic_tests:-0}))" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' @@ -445,6 +445,22 @@ assert_file_has_content checkout-test-union-add/union-add-test 'existing file fo assert_file_has_content checkout-test-union-add/union-add-test2 'another file for union add testing' echo "ok checkout union add" +cd ${test_tmpdir} +rm files -rf && mkdir files +mkdir files/worldwritable-dir +chmod a+w files/worldwritable-dir +$CMD_PREFIX ostree --repo=repo commit -b content-with-dir-world-writable --tree=dir=files +rm dir-co -rf +$CMD_PREFIX ostree --repo=repo checkout -U -H -M content-with-dir-world-writable dir-co +assert_file_has_mode dir-co/worldwritable-dir 775 +if ! is_bare_user_only_repo repo; then + rm dir-co -rf + $CMD_PREFIX ostree --repo=repo checkout -U -H content-with-dir-world-writable dir-co + assert_file_has_mode dir-co/worldwritable-dir 777 +fi +rm dir-co -rf +echo "ok checkout bareuseronly dir" + cd ${test_tmpdir} rm -rf shadow-repo mkdir shadow-repo From 64ab8334b7653562a9cd05ff4606efdf6b8331ac Mon Sep 17 00:00:00 2001 From: Anton Gerasimov Date: Thu, 1 Jun 2017 12:43:50 +0200 Subject: [PATCH 74/86] lib/sysroot: Add API to get pending/rollback for given stateroot This imports a function that is used in rpm-ostree, and it's also intended for use by https://github.com/advancedtelematic/aktualizr to display what deployment we're going to boot next after the reboot. Updated-by: Colin Walters Closes: #897 Approved by: OYTIS --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 1 + src/libostree/ostree-sysroot.c | 73 +++++++++++++++++++++++----- src/libostree/ostree-sysroot.h | 6 +++ src/ostree/ot-admin-builtin-status.c | 16 +++++- tests/admin-test.sh | 2 + 6 files changed, 84 insertions(+), 15 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 5e111a96..116c50e8 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -496,6 +496,7 @@ ostree_sysroot_write_deployments_with_options ostree_sysroot_write_origin_file ostree_sysroot_deploy_tree ostree_sysroot_get_merge_deployment +ostree_sysroot_query_deployments_for ostree_sysroot_origin_new_from_refspec OstreeSysrootSimpleWriteDeploymentFlags ostree_sysroot_simple_write_deployment diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 612eb8ac..b307548d 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -405,6 +405,7 @@ global: LIBOSTREE_2017.7 { global: ostree_sysroot_repo; + ostree_sysroot_query_deployments_for; } LIBOSTREE_2017.6; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index a3e9e75d..90868aae 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1107,6 +1107,63 @@ find_booted_deployment (OstreeSysroot *self, return TRUE; } +/** + * ostree_sysroot_query_deployments_for: + * @self: Sysroot + * @osname: (allow-none): "stateroot" name + * @out_pending: (out) (allow-none) (transfer full): The pending deployment + * @out_rollback: (out) (allow-none) (transfer full): The rollback deployment + * + * Find the pending and rollback deployments for @osname. Pass %NULL for @osname + * to use the booted deployment's osname. By default, pending deployment is the + * first deployment in the order that matches @osname, and @rollback will be the + * next one after the booted deployment, or the deployment after the pending if + * we're not looking at the booted deployment. + * + * Since: 2017.7 + */ +void +ostree_sysroot_query_deployments_for (OstreeSysroot *self, + const char *osname, + OstreeDeployment **out_pending, + OstreeDeployment **out_rollback) +{ + g_return_if_fail (osname != NULL || self->booted_deployment != NULL); + g_autoptr(OstreeDeployment) ret_pending = NULL; + g_autoptr(OstreeDeployment) ret_rollback = NULL; + + if (osname == NULL) + osname = ostree_deployment_get_osname (self->booted_deployment); + + gboolean found_booted = FALSE; + for (guint i = 0; i < self->deployments->len; i++) + { + OstreeDeployment *deployment = self->deployments->pdata[i]; + + /* Is this deployment booted? If so, note we're past the booted */ + if (self->booted_deployment != NULL && + ostree_deployment_equal (deployment, self->booted_deployment)) + { + found_booted = TRUE; + continue; + } + + /* Ignore deployments not for this osname */ + if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) + continue; + + if (!found_booted && !ret_pending) + ret_pending = g_object_ref (deployment); + else if (found_booted && !ret_rollback) + ret_rollback = g_object_ref (deployment); + } + if (out_pending) + *out_pending = g_steal_pointer (&ret_pending); + if (out_rollback) + *out_rollback = g_steal_pointer (&ret_rollback); +} + + /** * ostree_sysroot_get_merge_deployment: * @self: Sysroot @@ -1132,23 +1189,13 @@ ostree_sysroot_get_merge_deployment (OstreeSysroot *self, */ if (self->booted_deployment && g_strcmp0 (ostree_deployment_get_osname (self->booted_deployment), osname) == 0) - { return g_object_ref (self->booted_deployment); - } else { - guint i; - for (i = 0; i < self->deployments->len; i++) - { - OstreeDeployment *deployment = self->deployments->pdata[i]; - - if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) - continue; - - return g_object_ref (deployment); - } + g_autoptr(OstreeDeployment) pending = NULL; + ostree_sysroot_query_deployments_for (self, osname, &pending, NULL); + return g_steal_pointer (&pending); } - return NULL; } /** diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index e5969e9e..3d2446f9 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -187,6 +187,12 @@ gboolean ostree_sysroot_deployment_unlock (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +void ostree_sysroot_query_deployments_for (OstreeSysroot *self, + const char *osname, + OstreeDeployment **out_pending, + OstreeDeployment **out_rollback); + _OSTREE_PUBLIC OstreeDeployment *ostree_sysroot_get_merge_deployment (OstreeSysroot *self, const char *osname); diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 79621a1d..a437e7cf 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -88,6 +88,8 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro gboolean ret = FALSE; glnx_unref_object OstreeRepo *repo = NULL; OstreeDeployment *booted_deployment = NULL; + g_autoptr(OstreeDeployment) pending_deployment = NULL; + g_autoptr(OstreeDeployment) rollback_deployment = NULL; g_autoptr(GPtrArray) deployments = NULL; const int is_tty = isatty (1); const char *red_bold_prefix = is_tty ? "\x1b[31m\x1b[1m" : ""; @@ -110,6 +112,10 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro deployments = ostree_sysroot_get_deployments (sysroot); booted_deployment = ostree_sysroot_get_booted_deployment (sysroot); + if (booted_deployment) + ostree_sysroot_query_deployments_for (sysroot, NULL, &pending_deployment, + &rollback_deployment); + if (deployments->len == 0) { g_print ("No deployments.\n"); @@ -129,11 +135,17 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro origin = ostree_deployment_get_origin (deployment); - g_print ("%c %s %s.%d\n", + const char *deployment_status = ""; + if (deployment == pending_deployment) + deployment_status = " (pending)"; + else if (deployment == rollback_deployment) + deployment_status = " (rollback)"; + g_print ("%c %s %s.%d%s\n", deployment == booted_deployment ? '*' : ' ', ostree_deployment_get_osname (deployment), ostree_deployment_get_csum (deployment), - ostree_deployment_get_deployserial (deployment)); + ostree_deployment_get_deployserial (deployment), + deployment_status); if (version) g_print (" Version: %s\n", version); switch (unlocked) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index cc06fe6f..671fd905 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -42,6 +42,8 @@ ${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos new_mtime=$(stat -c '%.Y' sysroot/ostree/deploy) assert_not_streq "${orig_mtime}" "${new_mtime}" ${CMD_PREFIX} ostree admin status | tee status.txt +assert_not_file_has_content status.txt "pending" +assert_not_file_has_content status.txt "rollback" validate_bootloader echo "ok deploy command" From 584735b1c9a0b4ef62f7a8ed145447b27c3a15c9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 13 Jun 2017 14:52:46 -0400 Subject: [PATCH 75/86] build-sys: post-release version bump Per request by flatpak. Closes: #928 Approved by: alexlarsson --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c60806f4..0efe6530 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too m4_define([year_version], [2017]) -m4_define([release_version], [6]) +m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) From 0e6d23835baedc472d3967105ba1f0d9f25068a8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 14 Jun 2017 11:31:52 -0400 Subject: [PATCH 76/86] lib/sysroot: Add some g_prefix_error() for ostree_sysroot_cleanup() We saw this fail in a CI run. We've been trying to add strategic error prefixing as a general rule, and this specific instance may help debug. Closes: #929 Approved by: pwithnall --- src/libostree/ostree-sysroot-cleanup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index c0b8bf07..79663b66 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -505,10 +505,10 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, g_return_val_if_fail (self->loaded, FALSE); if (!cleanup_other_bootversions (self, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Cleaning bootversions"); if (!cleanup_old_deployments (self, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Cleaning deployments"); OstreeRepo *repo = ostree_sysroot_repo (self); if (!generate_deployment_refs (self, repo, @@ -516,12 +516,12 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot *self, self->subbootversion, self->deployments, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Generating deployment refs"); if (do_prune_repo) { if (!prune_repo (repo, cancellable, error)) - return FALSE; + return glnx_prefix_error (error, "Pruning repo"); } return TRUE; From 9529e8d43579aaf6f6ebb90516385a89aaff34ff Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 14 Jun 2017 17:55:11 -0400 Subject: [PATCH 77/86] lib/pull: Extend BAREUSERONLY_FILES flag to HTTP requests For the flatpak PR: https://github.com/flatpak/flatpak/pull/849 It's really more convenient if this works for HTTP pulls as well, since flatpak does various types of pulling, and we can just set the flag everywhere. Further, we might as well reject the content as early as possible. Closes: #930 Approved by: alexlarsson --- src/libostree/ostree-repo-pull.c | 23 ++++++++++++-------- tests/pull-test.sh | 36 +++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 2e93f90b..704a68d2 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -871,7 +871,12 @@ content_fetch_on_complete (GObject *object, checksum_obj = ostree_object_to_string (checksum, objtype); g_debug ("fetch of %s complete", checksum_obj); - if (pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) + /* If we're mirroring and writing into an archive repo, we can directly copy + * the content rather than paying the cost of exploding it, checksumming, and + * re-gzip. + */ + if (pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 + && !pull_data->is_bareuseronly_files) { gboolean have_object; if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum, @@ -903,16 +908,22 @@ content_fetch_on_complete (GObject *object, } /* Also, delete it now that we've opened it, we'll hold - * a reference to the fd. If we fail to write later, then + * a reference to the fd. If we fail to validate or write, then * the temp space will be cleaned up. */ (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); + if (!validate_bareuseronly_mode (pull_data, + checksum, + g_file_info_get_attribute_uint32 (file_info, "unix::mode"), + error)) + goto out; + if (!ostree_raw_file_to_content_stream (file_in, file_info, xattrs, &object_input, &length, cancellable, error)) goto out; - + pull_data->n_outstanding_content_write_requests++; ostree_repo_write_content_async (pull_data->repo, checksum, object_input, length, @@ -3145,12 +3156,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->disable_static_deltas = TRUE; } - else if (pull_data->is_bareuseronly_files) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't use bareuseronly-files with non-local origin repo"); - goto out; - } /* We can't use static deltas if pulling into an archive-z2 repo. */ if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index f1ebca4c..9f74c0d8 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..21" +echo "1..23" # Try both syntaxes repo_init --no-gpg-verify @@ -79,12 +79,34 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo pull origin main ${CMD_PREFIX} ostree --repo=mirrorrepo fsck echo "ok pull (refuses deltas)" -if ${CMD_PREFIX} ostree --repo=mirrorrepo \ - pull origin main --bareuseronly-files 2>err.txt; then - assert_not_reached "--bareuseronly-files unexpectedly succeeded" -fi -assert_file_has_content err.txt 'bareuseronly-files with non-local' -echo "ok pull (refuses bareuseronly)" +cd ${test_tmpdir} +rm mirrorrepo/refs/remotes/* -rf +${CMD_PREFIX} ostree --repo=mirrorrepo prune --refs-only +${CMD_PREFIX} ostree --repo=mirrorrepo pull --bareuseronly-files origin main +echo "ok pull (bareuseronly, safe)" + +rm checkout-origin-main -rf +$OSTREE --repo=ostree-srv/gnomerepo checkout main checkout-origin-main +cat > statoverride.txt < checkout-origin-main/some-setuid +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b content-with-suid --statoverride=statoverride.txt --tree=dir=checkout-origin-main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +# Verify we reject it both when unpacking and when mirroring +for flag in "" "--mirror"; do + if ${CMD_PREFIX} ostree --repo=mirrorrepo pull ${flag} --bareuseronly-files origin content-with-suid 2>err.txt; then + assert_not_reached "pulled unsafe bareuseronly" + fi + assert_file_has_content err.txt 'object.*\.file: invalid mode.*with bits 040.*' +done +echo "ok pull (bareuseronly, unsafe)" + +cd ${test_tmpdir} +rm mirrorrepo/refs/remotes/* -rf +${CMD_PREFIX} ostree --repo=mirrorrepo prune --refs-only +${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror --bareuseronly-files origin main +echo "ok pull (bareuseronly mirror)" cd ${test_tmpdir} rm mirrorrepo/refs/remotes/* -rf From 3e3a0f0766918e4750d136bcb2782482135d0b49 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Thu, 15 Jun 2017 18:00:27 +0900 Subject: [PATCH 78/86] ostreee-version.h.in: Added Since: version annotations This is especially interesting for the versioning symbols themselves, as it is an indicator of when applications using introspection information can start to use a symbol in the library to check if they have a recent enough version of OSTree to use. Closes: #932 Approved by: cgwalters --- src/libostree/ostree-version.h.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libostree/ostree-version.h.in b/src/libostree/ostree-version.h.in index 7d775cc2..3f4d0fb7 100644 --- a/src/libostree/ostree-version.h.in +++ b/src/libostree/ostree-version.h.in @@ -32,6 +32,8 @@ * OSTREE_YEAR_VERSION: * * ostree year version component (e.g. 2017 if %OSTREE_VERSION is 2017.2) + * + * Since: 2017.4 */ #define OSTREE_YEAR_VERSION (@YEAR_VERSION@) @@ -39,6 +41,8 @@ * OSTREE_RELEASE_VERSION: * * ostree release version component (e.g. 2 if %OSTREE_VERSION is 2017.2) + * + * Since: 2017.4 */ #define OSTREE_RELEASE_VERSION (@RELEASE_VERSION@) @@ -46,6 +50,8 @@ * OSTREE_VERSION * * ostree version. + * + * Since: 2017.4 */ #define OSTREE_VERSION (@VERSION@) @@ -54,6 +60,8 @@ * * ostree version, encoded as a string, useful for printing and * concatenation. + * + * Since: 2017.4 */ #define OSTREE_VERSION_S "@VERSION@" @@ -65,6 +73,8 @@ * * ostree version, encoded as an hexadecimal number, useful for * integer comparisons. + * + * Since: 2017.4 */ #define OSTREE_VERSION_HEX \ (OSTREE_ENCODE_VERSION (OSTREE_YEAR_VERSION, OSTREE_RELEASE_VERSION)) @@ -76,6 +86,8 @@ * * Compile-time version checking. Evaluates to %TRUE if the version * of ostree is equal or greater than the required one. + * + * Since: 2017.4 */ #define OSTREE_CHECK_VERSION(year,release) \ (OSTREE_YEAR_VERSION > (year) || \ From 2bab43fb224edc5b0bfacc79881e387b8de67c4a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 14 Jun 2017 21:44:04 -0400 Subject: [PATCH 79/86] lib: Split symbol versioning into -released and -devel So far a lot of submitted PR have added symbols into the first section. Split the file into `-released` and `-devel` to make this more obvious. To further enforce things, we hardcode a checksum of the `-released` file in `test-symbols.sh`. Only release commits should update that checksum. Did you notice I like checksums? Closes: #931 Approved by: pwithnall --- Makefile-libostree.am | 24 ++++++------- configure.ac | 4 ++- src/libostree/libostree-devel.sym | 35 +++++++++++++++++++ .../{libostree.sym => libostree-released.sym} | 28 ++++----------- tests/test-symbols.sh | 16 +++++++-- 5 files changed, 69 insertions(+), 38 deletions(-) create mode 100644 src/libostree/libostree-devel.sym rename src/libostree/{libostree.sym => libostree-released.sym} (95%) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 7f2e2a4a..3c2b6237 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -155,24 +155,24 @@ libostree_1_la_SOURCES += \ $(NULL) endif +symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym \ + $(top_srcdir)/src/libostree/libostree-devel.sym +if ENABLE_EXPERIMENTAL_API +symbol_files += $(top_srcdir)/src/libostree/libostree-experimental.sym +endif +# http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html +wl_versionscript_arg = -Wl,--version-script= +EXTRA_DIST += $(symbol_files) + + libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \ $(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) $(OT_DEP_OPENSSL_CFLAGS) \ -fvisibility=hidden '-D_OSTREE_PUBLIC=__attribute__((visibility("default"))) extern' -libostree_1_la_LDFLAGS = -version-number 1:0:0 -Bsymbolic-functions -Wl,--version-script=$(top_srcdir)/src/libostree/libostree.sym +libostree_1_la_LDFLAGS = -version-number 1:0:0 -Bsymbolic-functions $(addprefix $(wl_versionscript_arg),$(symbol_files)) libostree_1_la_LIBADD = libotutil.la libglnx.la libbsdiff.la libostree-kernel-args.la $(OT_INTERNAL_GIO_UNIX_LIBS) $(OT_INTERNAL_GPGME_LIBS) \ $(OT_DEP_LZMA_LIBS) $(OT_DEP_ZLIB_LIBS) $(OT_DEP_OPENSSL_LIBS) libostree_1_la_LIBADD += $(bupsplitpath) -EXTRA_libostree_1_la_DEPENDENCIES = $(top_srcdir)/src/libostree/libostree.sym - -EXTRA_DIST += \ - src/libostree/libostree.sym \ - src/libostree/libostree-experimental.sym \ - $(NULL) - -if ENABLE_EXPERIMENTAL_API -libostree_1_la_LDFLAGS += -Wl,--version-script=$(top_srcdir)/src/libostree/libostree-experimental.sym -EXTRA_libostree_1_la_DEPENDENCIES += $(top_srcdir)/src/libostree/libostree-experimental.sym -endif +EXTRA_libostree_1_la_DEPENDENCIES = $(symbol_files) if USE_LIBARCHIVE libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) diff --git a/configure.ac b/configure.ac index 0efe6530..cf8a01b8 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,7 @@ AC_PREREQ([2.63]) -dnl If incrementing the version here, remember to update libostree.sym too +dnl If doing a final release, remember to follow the instructions to +dnl update libostree-released.sym from libostree-devel.sym, and update the checksum +dnl in test-symbols.sh m4_define([year_version], [2017]) m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym new file mode 100644 index 00000000..6616432b --- /dev/null +++ b/src/libostree/libostree-devel.sym @@ -0,0 +1,35 @@ +/* DEVEL symbol file - add new symbols here + Copyright (C) 2016 Colin Walters + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +***/ + +/* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2017.7 { +global: + ostree_sysroot_repo; + ostree_sysroot_query_deployments_for; +} LIBOSTREE_2017.6; + +/* Stub section for the stable release *after* this development one; don't + * edit this other than to update the last number. This is just a copy/paste + * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION + * with whatever the next version with new symbols will be. +LIBOSTREE_2017.$NEWVERSION { +global: + someostree_symbol_deleteme; +} LIBOSTREE_2017.$LASTSTABLE; +*/ diff --git a/src/libostree/libostree.sym b/src/libostree/libostree-released.sym similarity index 95% rename from src/libostree/libostree.sym rename to src/libostree/libostree-released.sym index b307548d..b201dcab 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree-released.sym @@ -1,4 +1,6 @@ -/*** +/* Released symbol file: DO NOT EDIT except in release commits! + Edit libostree-devel.sym instead. + Copyright (C) 2016 Colin Walters This library is free software; you can redistribute it and/or @@ -17,6 +19,9 @@ Boston, MA 02111-1307, USA. ***/ +/* DO NOT EDIT this file - use libostree-devel.sym instead. Release + * commits will move symbols from libostree-devel.sym to libostree-released.sym. + */ /* Retroactively make all of these symbols 2016.3, which is @@ -396,24 +401,3 @@ global: ostree_async_progress_get_variant; ostree_async_progress_set_variant; } LIBOSTREE_2017.4; - -/* NOTE NOTE NOTE - * Versions above here are released. Only add symbols below this line. - * NOTE NOTE NOTE - */ - -LIBOSTREE_2017.7 { -global: - ostree_sysroot_repo; - ostree_sysroot_query_deployments_for; -} LIBOSTREE_2017.6; - -/* Stub section for the stable release *after* this development one; don't - * edit this other than to update the last number. This is just a copy/paste - * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION - * with whatever the next version with new symbols will be. -LIBOSTREE_2017.$NEWVERSION { -global: - someostree_symbol_deleteme; -} LIBOSTREE_2017.$LASTSTABLE; -*/ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 54f469fb..9018e6f3 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -17,10 +17,12 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -euo pipefail +set -xeuo pipefail -echo '1..2' +echo '1..3' +released_syms=${G_TEST_SRCDIR}/src/libostree/libostree-released.sym +devel_syms=${G_TEST_SRCDIR}/src/libostree/libostree-devel.sym if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "experimental"; then experimental_sym="${G_TEST_SRCDIR}/src/libostree/libostree-experimental.sym" experimental_sections="${G_TEST_SRCDIR}/apidoc/ostree-experimental-sections.txt" @@ -30,7 +32,7 @@ else fi echo "Verifying all expected symbols are actually exported..." -grep --no-filename ' ostree_[A-Za-z0-9_]*;' ${G_TEST_SRCDIR}/src/libostree/libostree.sym $experimental_sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt +grep --no-filename ' ostree_[A-Za-z0-9_]*;' ${released_syms} ${devel_syms} ${experimental_sym} | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9A-Z_.]*,\1,' |sort -u > found-symbols.txt diff -u expected-symbols.txt found-symbols.txt echo "ok exports" @@ -43,3 +45,11 @@ grep --no-filename '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt $exper diff -u expected-documented.txt found-documented.txt echo 'ok documented symbols' + +# ONLY update this checksum in release commits! +cat > released-sha256.txt < Date: Fri, 16 Jun 2017 10:36:28 -0400 Subject: [PATCH 80/86] checkout: Fix SELinux policy labeling when recursing The code here tried to truncate the string to the previous length, but that doesn't work when recursing, since further calls change the length. What actually ended up happening was the string would get corrupted after the first level of recursion. Closes: #936 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 12 ++++++------ tests/installed/itest-deploy-selinux.sh | 7 ++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 4b14dcdb..af5c021f 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -693,9 +693,9 @@ checkout_tree_at_recurse (OstreeRepo *self, g_autoptr(GVariant) contents_csum_v = NULL; while (g_variant_iter_loop (&viter, "(&s@ay)", &fname, &contents_csum_v)) { - const size_t namelen = strlen (fname); + const size_t origlen = selabel_path_buf ? selabel_path_buf->len : 0; if (selabel_path_buf) - g_string_append_len (selabel_path_buf, fname, namelen); + g_string_append (selabel_path_buf, fname); char tmp_checksum[OSTREE_SHA256_STRING_LEN+1]; _ostree_checksum_inplace_from_bytes_v (contents_csum_v, tmp_checksum); @@ -707,7 +707,7 @@ checkout_tree_at_recurse (OstreeRepo *self, return FALSE; if (selabel_path_buf) - g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); + g_string_truncate (selabel_path_buf, origlen); } contents_csum_v = NULL; /* iter_loop freed it */ } @@ -722,10 +722,10 @@ checkout_tree_at_recurse (OstreeRepo *self, while (g_variant_iter_loop (&viter, "(&s@ay@ay)", &dname, &subdirtree_csum_v, &subdirmeta_csum_v)) { - const size_t namelen = strlen (dname); + const size_t origlen = selabel_path_buf ? selabel_path_buf->len : 0; if (selabel_path_buf) { - g_string_append_len (selabel_path_buf, dname, namelen); + g_string_append (selabel_path_buf, dname); g_string_append_c (selabel_path_buf, '/'); } @@ -740,7 +740,7 @@ checkout_tree_at_recurse (OstreeRepo *self, return FALSE; if (selabel_path_buf) - g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); + g_string_truncate (selabel_path_buf, origlen); } } diff --git a/tests/installed/itest-deploy-selinux.sh b/tests/installed/itest-deploy-selinux.sh index c4965f87..f4fccc6d 100755 --- a/tests/installed/itest-deploy-selinux.sh +++ b/tests/installed/itest-deploy-selinux.sh @@ -12,7 +12,12 @@ ostree admin deploy --karg-proc-cmdline ${host_refspec} new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1 # A set of files that have a variety of security contexts -for file in fstab passwd exports hostname sysctl.conf; do +for file in fstab passwd exports hostname sysctl.conf /etc/yum.repos.d \ + /etc/NetworkManager/dispatcher.d/hook-network-manager; do + if ! test -e ${file}; then + continue + fi + current=$(cd /etc && ls -Z ${file}) new=$(cd ${new_deployment_path}/etc && ls -Z ${file}) assert_streq "${current}" "${new}" From 73ba3eb686ef86cea1c4563303e44df87d73e6c6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 16 Jun 2017 10:07:44 +0200 Subject: [PATCH 81/86] pull: When mirroring, only replace summary if we're doing a full mirror We're hitting this in flathub, where we have a bunch of local builds, but we also mirror a few refs from the gnome runtime repo into it. Its fixable by re-doing the summary, but for a short time the wrong version is visible. Fixes https://github.com/ostreedev/ostree/issues/846 Closes: #935 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- tests/pull-test.sh | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 704a68d2..2c87fd60 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3518,7 +3518,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - if (pull_data->is_mirror && pull_data->summary_data) + if (pull_data->is_mirror && pull_data->summary_data && !refs_to_fetch && !configured_branches) { GLnxFileReplaceFlags replaceflag = pull_data->repo->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : 0; diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 9f74c0d8..24eb16a0 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..23" +echo "1..25" # Try both syntaxes repo_init --no-gpg-verify @@ -57,6 +57,33 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo fsck $OSTREE show main >/dev/null echo "ok pull mirror" +mkdir otherbranch +echo someothercontent > otherbranch/someothercontent +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b otherbranch --tree=dir=otherbranch +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +rm mirrorrepo -rf +# All refs +ostree_repo_init mirrorrepo --mode=archive-z2 +${CMD_PREFIX} ostree --repo=mirrorrepo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo +${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror origin +${CMD_PREFIX} ostree --repo=mirrorrepo fsck +for ref in main otherbranch; do + ${CMD_PREFIX} ostree --repo=mirrorrepo rev-parse $ref +done +echo "ok pull mirror (all refs)" + +rm mirrorrepo -rf +ostree_repo_init mirrorrepo --mode=archive-z2 +${CMD_PREFIX} ostree --repo=mirrorrepo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo +# Generate a summary in the mirror +${CMD_PREFIX} ostree --repo=mirrorrepo summary -u +summarysig=$(sha256sum < mirrorrepo/summary | cut -f 1 -d ' ') +# Mirror subset of refs: https://github.com/ostreedev/ostree/issues/846 +${CMD_PREFIX} ostree --repo=mirrorrepo pull --mirror origin main +newsummarysig=$(sha256sum < mirrorrepo/summary | cut -f 1 -d ' ') +assert_streq ${summarysig} ${newsummarysig} +echo "ok pull mirror (ref subset with summary)" + cd ${test_tmpdir} rm checkout-origin-main -rf $OSTREE --repo=ostree-srv/gnomerepo checkout main checkout-origin-main From fb2c3c1db3c655c9a96442f6706017a4d0b48080 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 16 Jun 2017 11:26:43 -0400 Subject: [PATCH 82/86] tests: Fix previous commit for selinux testing I only checked the test passed, I didn't read the output closely, and made it succeed without testing anything. Fix the absolute/relative `/etc` references. Closes: #937 Approved by: jlebon --- tests/installed/itest-deploy-selinux.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/installed/itest-deploy-selinux.sh b/tests/installed/itest-deploy-selinux.sh index f4fccc6d..7c26aad0 100755 --- a/tests/installed/itest-deploy-selinux.sh +++ b/tests/installed/itest-deploy-selinux.sh @@ -12,9 +12,9 @@ ostree admin deploy --karg-proc-cmdline ${host_refspec} new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1 # A set of files that have a variety of security contexts -for file in fstab passwd exports hostname sysctl.conf /etc/yum.repos.d \ - /etc/NetworkManager/dispatcher.d/hook-network-manager; do - if ! test -e ${file}; then +for file in fstab passwd exports hostname sysctl.conf yum.repos.d \ + NetworkManager/dispatcher.d/hook-network-manager; do + if ! test -e /etc/${file}; then continue fi From 07dc33ca4a3f2a815143ddab09eb9a7980716c87 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 19 Jun 2017 11:06:30 +0200 Subject: [PATCH 83/86] static delta apply: Work on bare-user-only repos Flatpak make check is failing when applying a static delta to a bare-user-only repo due to an assert. The fix is to add bare-user-only to the assert check. Closes: #940 Approved by: giuseppe --- src/libostree/ostree-repo-static-delta-processing.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 2280dec9..ea157e77 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -690,7 +690,8 @@ dispatch_open (OstreeRepo *repo, if (!state->stats_only) { g_assert (repo->mode == OSTREE_REPO_MODE_BARE || - repo->mode == OSTREE_REPO_MODE_BARE_USER); + repo->mode == OSTREE_REPO_MODE_BARE_USER || + repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY); } if (!open_output_target (state, cancellable, error)) From 22e753176e62d3c7377ccb2e51108191a4b41d9d Mon Sep 17 00:00:00 2001 From: David Shea Date: Sat, 17 Jun 2017 12:08:41 -0400 Subject: [PATCH 84/86] lib/repo: Fix annotations for out parameters Change the annotation of the out parameters on ostree_repo_load_file from `(allow-none)` to `(optional) (nullable)`. `allow-none` is ambiguous, since these parameters can be both NULL on input and set to NULL on return. Closes: #939 Approved by: cgwalters --- src/libostree/ostree-repo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 49251284..e556e464 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2618,9 +2618,9 @@ _ostree_repo_read_bare_fd (OstreeRepo *self, * ostree_repo_load_file: * @self: Repo * @checksum: ASCII SHA256 checksum - * @out_input: (out) (allow-none): File content - * @out_file_info: (out) (allow-none): File information - * @out_xattrs: (out) (allow-none): Extended attributes + * @out_input: (out) (optional) (nullable): File content + * @out_file_info: (out) (optional) (nullable): File information + * @out_xattrs: (out) (optional) (nullable): Extended attributes * @cancellable: Cancellable * @error: Error * From a45dc0fd0bd3675471f4f18a77df0a5785f54ebb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Jun 2017 10:23:21 -0400 Subject: [PATCH 85/86] build-sys: Add "release build" flag, use for symbol versioning I was trying to do a release and move the symbols from `-devel.sym` into `-release.sym`, but it turns out that at least GNU binutils `ld` treats an empty version script as a syntax error. Fix this by adding a "release build" flag, and only include `-devel` in non-release builds. This would also make it easier to inject that flag into our `.pc` and `ostree-version.h` and `ostree --version` metadata, but I didn't do that yet. EDIT: Turns out a simpler fix is just to add an empty section. However I kept this commit since it's a useful sanity check for whether we should include `-devel.sym` in builds, and we may want to inject the metadata later. Closes: #942 Approved by: jlebon --- Makefile-libostree.am | 7 ++++--- configure.ac | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 3c2b6237..61ad1f4a 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -155,8 +155,10 @@ libostree_1_la_SOURCES += \ $(NULL) endif -symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym \ - $(top_srcdir)/src/libostree/libostree-devel.sym +symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym +if BUILDOPT_IS_DEVEL_BUILD +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +endif if ENABLE_EXPERIMENTAL_API symbol_files += $(top_srcdir)/src/libostree/libostree-experimental.sym endif @@ -164,7 +166,6 @@ endif wl_versionscript_arg = -Wl,--version-script= EXTRA_DIST += $(symbol_files) - libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \ $(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) $(OT_DEP_OPENSSL_CFLAGS) \ -fvisibility=hidden '-D_OSTREE_PUBLIC=__attribute__((visibility("default"))) extern' diff --git a/configure.ac b/configure.ac index cf8a01b8..448c7f4e 100644 --- a/configure.ac +++ b/configure.ac @@ -1,12 +1,13 @@ AC_PREREQ([2.63]) dnl If doing a final release, remember to follow the instructions to dnl update libostree-released.sym from libostree-devel.sym, and update the checksum -dnl in test-symbols.sh +dnl in test-symbols.sh, and also set is_release_build=yes below. Then make +dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2017]) m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) - AC_INIT([libostree], [package_version], [walters@verbum.org]) +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) @@ -458,6 +459,8 @@ AS_IF([test x$enable_experimental_api = xyes], OSTREE_FEATURES="$OSTREE_FEATURES experimental"] ) AM_CONDITIONAL([ENABLE_EXPERIMENTAL_API],[test x$enable_experimental_api = xyes]) +AM_CONDITIONAL([BUILDOPT_IS_DEVEL_BUILD],[test x$is_release_build != xyes]) +AM_COND_IF([BUILDOPT_IS_DEVEL_BUILD], release_build_type=devel, release_build_type=release) AC_CONFIG_FILES([ Makefile @@ -468,7 +471,7 @@ src/libostree/ostree-version.h AC_OUTPUT echo " - libOSTree $VERSION + libOSTree $VERSION ($release_build_type) =============== From 6729b7c2649a2c78bfbafc69a2b902f635275330 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Jun 2017 10:34:35 -0400 Subject: [PATCH 86/86] Release 2017.7 Closes: #942 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 5 ----- src/libostree/libostree-released.sym | 10 ++++++++++ tests/test-symbols.sh | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 448c7f4e..7946f4fd 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ m4_define([year_version], [2017]) m4_define([release_version], [7]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 6616432b..01f182f6 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,11 +18,6 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2017.7 { -global: - ostree_sysroot_repo; - ostree_sysroot_query_deployments_for; -} LIBOSTREE_2017.6; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index b201dcab..5fc8a9b1 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -401,3 +401,13 @@ global: ostree_async_progress_get_variant; ostree_async_progress_set_variant; } LIBOSTREE_2017.4; + +LIBOSTREE_2017.7 { +global: + ostree_sysroot_repo; + ostree_sysroot_query_deployments_for; +} LIBOSTREE_2017.6; + +/* NOTE: Only add more content here in release commits! See the + * comments at the top of this file. + */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 9018e6f3..d22231d0 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -48,7 +48,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <