From b9df96db8bd5bc40f90a9006c9aa8cbcf01fbc41 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 11 Apr 2017 17:22:54 -0400 Subject: [PATCH 01/94] pull: Support deltas for explicit commits I think the majority of OSTree usage calls pull with refs, not explicit commits. We even added special "override syntax" with `@` (e.g. `ostree pull foo@ab12c34`) as a hybrid. However, some users may want to still pull explicit commits for whatever reason. The old static delta logic looked at the previous commit of the ref. However, in https://github.com/ostreedev/ostree/pull/710 we enhanced the logic to look at all local commits. It's now a lot more natural to teach the delta logic to support revisions, e.g. `ostree pull someorigin ab12c34`. This also fixes the problem that before, `--require-static-deltas` was completely ignored when processing revisions. This is a nontrivial refactoring of the logic, but the end result feels a lot more readable to me. Closes: https://github.com/ostreedev/ostree/issues/783 Closes: #787 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 214 +++++++++++++++++++++---------- tests/pull-test.sh | 35 ++++- tests/test-delta.sh | 4 +- 3 files changed, 175 insertions(+), 78 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a7a3a5b0..3f5b771c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -68,6 +68,7 @@ typedef struct { gboolean gpg_verify; gboolean require_static_deltas; + gboolean disable_static_deltas; gboolean gpg_verify_summary; gboolean has_tombstone_commits; @@ -1879,10 +1880,15 @@ process_one_static_delta (OtPullData *pull_data, /* Loop over the static delta data we got from the summary, * and find the newest commit for @out_from_revision that * goes to @to_revision. + * + * Additionally, @out_have_scratch_delta will be set to %TRUE + * if there is a %NULL → @to_revision delta, also known as + * a "from scratch" delta. */ static gboolean get_best_static_delta_start_for (OtPullData *pull_data, const char *to_revision, + gboolean *out_have_scratch_delta, char **out_from_revision, GCancellable *cancellable, GError **error) @@ -1897,6 +1903,8 @@ get_best_static_delta_start_for (OtPullData *pull_data, g_assert (pull_data->summary_deltas_checksums != NULL); g_hash_table_iter_init (&hiter, pull_data->summary_deltas_checksums); + *out_have_scratch_delta = FALSE; + /* Loop over all deltas known from the summary file, * finding ones which go to to_revision */ while (g_hash_table_iter_next (&hiter, &hkey, &hvalue)) @@ -1915,6 +1923,8 @@ get_best_static_delta_start_for (OtPullData *pull_data, if (cur_from_rev) g_ptr_array_add (candidates, g_steal_pointer (&cur_from_rev)); + else + *out_have_scratch_delta = TRUE; } /* Loop over our candidates, find the newest one */ @@ -1963,6 +1973,16 @@ typedef struct { char *to_revision; } FetchDeltaSuperData; +static void +set_required_deltas_error (GError **error, + const char *from_revision, + const char *to_revision) +{ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Static deltas required, but none found for %s to %s", + from_revision, to_revision); +} + static void on_superblock_fetched (GObject *src, GAsyncResult *res, @@ -1984,14 +2004,11 @@ on_superblock_fetched (GObject *src, { if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) goto out; - g_clear_error (&local_error); if (pull_data->require_static_deltas) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Static deltas required, but none found for %s to %s", - from_revision, to_revision); + set_required_deltas_error (error, from_revision, to_revision); goto out; } @@ -2566,6 +2583,117 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **e return TRUE; } +/* Start a request for a static delta */ +static void +initiate_delta_request (OtPullData *pull_data, + const char *from_revision, + const char *to_revision) +{ + g_autofree char *delta_name = + _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision); + FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1); + fdata->pull_data = pull_data; + fdata->from_revision = g_strdup (from_revision); + fdata->to_revision = g_strdup (to_revision); + + _ostree_fetcher_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, 0, + OSTREE_MAX_METADATA_SIZE, + 0, pull_data->cancellable, + on_superblock_fetched, fdata); + pull_data->n_outstanding_metadata_fetches++; + pull_data->n_requested_metadata++; +} + +/* @ref - Optional ref name + * @to_revision: Target commit revision we want to fetch + * + * Start a request for either a ref or a commit. In the + * ref case, we know both the name and the target commit. + * + * This function primarily handles the semantics around + * `disable_static_deltas` and `require_static_deltas`. + */ +static gboolean +initiate_request (OtPullData *pull_data, + const char *ref, + const char *to_revision, + GError **error) +{ + g_autofree char *delta_from_revision = NULL; + + /* Are deltas disabled? OK, just start an object fetch and be done */ + if (pull_data->disable_static_deltas) + { + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); + return TRUE; + } + + /* If we have a summary, we can use the newer logic */ + if (pull_data->summary) + { + gboolean have_scratch_delta = FALSE; + + /* Look for a delta to @to_revision in the summary data */ + if (!get_best_static_delta_start_for (pull_data, to_revision, + &have_scratch_delta, &delta_from_revision, + pull_data->cancellable, error)) + return FALSE; + + if (delta_from_revision) /* Did we find a delta FROM commit? */ + initiate_delta_request (pull_data, delta_from_revision, to_revision); + else if (have_scratch_delta) /* No delta FROM, do we have a scratch? */ + initiate_delta_request (pull_data, NULL, to_revision); + else if (pull_data->require_static_deltas) /* No deltas found; are they required? */ + { + set_required_deltas_error (error, ref, to_revision); + return FALSE; + } + else /* No deltas, fall back to object fetches. */ + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); + } + else if (ref != NULL) + { + /* Are we doing a delta via a ref? In that case we can fall back to the older + * logic of just using the current tip of the ref as a delta FROM source. */ + if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, + &delta_from_revision, error)) + return FALSE; + + /* Determine whether the from revision we have is partial; this + * can happen if e.g. one uses `ostree pull --commit-metadata-only`. + * This mirrors the logic in get_best_static_delta_start_for(). + */ + if (delta_from_revision) + { + OstreeRepoCommitState from_commitstate; + + if (!ostree_repo_load_commit (pull_data->repo, delta_from_revision, NULL, + &from_commitstate, error)) + return FALSE; + + /* Was it partial? Then we can't use it. */ + if (commitstate_is_partial (pull_data, from_commitstate)) + g_clear_pointer (&delta_from_revision, g_free); + } + + /* This is similar to the below, except we *might* use the previous + * commit, or we might do a scratch delta first. + */ + initiate_delta_request (pull_data, delta_from_revision ?: NULL, to_revision); + } + else + { + /* Legacy path without a summary file - let's try a scratch delta, if that + * doesn't work, it'll drop down to object requests. + */ + initiate_delta_request (pull_data, NULL, to_revision); + } + + return TRUE; +} + /* ------------------------------------------------------------------------------------------ * Below is the libsoup-invariant API; these should match * the stub functions in the #else clause @@ -2631,7 +2759,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autofree char **refs_to_fetch = NULL; g_autofree char **override_commit_ids = NULL; GSource *update_timeout = NULL; - gboolean disable_static_deltas = FALSE; gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; const char *url_override = NULL; @@ -2653,7 +2780,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, opt_gpg_verify_summary_set = g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); - (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + (void) g_variant_lookup (options, "disable-static-deltas", "b", &pull_data->disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas); (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); @@ -2673,7 +2800,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, for (i = 0; dirs_to_pull != NULL && dirs_to_pull[i] != NULL; i++) g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE); - g_return_val_if_fail (!(disable_static_deltas && pull_data->require_static_deltas), FALSE); + g_return_val_if_fail (!(pull_data->disable_static_deltas && pull_data->require_static_deltas), FALSE); /* We only do dry runs with static deltas, because we don't really have any * in-advance information for bare fetches. @@ -2952,7 +3079,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, * exact object files are copied. */ if (pull_data->remote_repo_local && !pull_data->require_static_deltas) - disable_static_deltas = TRUE; + pull_data->disable_static_deltas = TRUE; /* We can't use static deltas if pulling into an archive-z2 repo. */ if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2) @@ -2963,13 +3090,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, "Can't use static deltas in an archive repo"); goto out; } - disable_static_deltas = TRUE; + pull_data->disable_static_deltas = TRUE; } /* It's not efficient to use static deltas if all we want is the commit * metadata. */ if (pull_data->is_commit_only) - disable_static_deltas = TRUE; + pull_data->disable_static_deltas = TRUE; pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); @@ -3239,78 +3366,23 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->legacy_transaction_resuming) g_debug ("resuming legacy transaction"); + /* Initiate requests for explicit commit revisions */ g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *commit = value; - queue_scan_one_metadata_object (pull_data, commit, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); + if (!initiate_request (pull_data, NULL, commit, error)) + goto out; } + /* Initiate requests for refs */ g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { - g_autofree char *from_revision = NULL; const char *ref = key; const char *to_revision = value; - gboolean have_valid_from_commit = TRUE; - - /* If we have a summary, find the latest local commit we have - * to use as a from revision for static deltas. - */ - if (pull_data->summary) - { - if (!get_best_static_delta_start_for (pull_data, to_revision, &from_revision, - cancellable, error)) - goto out; - } - else - { - if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, - &from_revision, error)) - goto out; - - /* Determine whether the from revision we have is partial; this - * can happen if e.g. one uses `ostree pull --commit-metadata-only`. - * This mirrors the logic in get_best_static_delta_start_for(). - */ - if (from_revision) - { - OstreeRepoCommitState from_commitstate; - - if (!ostree_repo_load_commit (pull_data->repo, from_revision, NULL, - &from_commitstate, error)) - goto out; - - /* Was it partial? OK, we can't use it. */ - if (commitstate_is_partial (pull_data, from_commitstate)) - have_valid_from_commit = FALSE; - } - } - - if (!disable_static_deltas && - have_valid_from_commit && - (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0)) - { - g_autofree char *delta_name = - _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision); - FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1); - fdata->pull_data = pull_data; - fdata->from_revision = g_strdup (from_revision); - fdata->to_revision = g_strdup (to_revision); - - _ostree_fetcher_request_to_membuf (pull_data->fetcher, - pull_data->content_mirrorlist, - delta_name, 0, - OSTREE_MAX_METADATA_SIZE, - 0, pull_data->cancellable, - on_superblock_fetched, fdata); - pull_data->n_outstanding_metadata_fetches++; - pull_data->n_requested_metadata++; - } - else - { - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0); - } + if (!initiate_request (pull_data, ref, to_revision, error)) + goto out; } if (pull_data->progress) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index f2486c31..f81d8023 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -35,7 +35,7 @@ function verify_initial_contents() { assert_file_has_content baz/cow '^moo$' } -echo "1..15" +echo "1..16" # Try both syntaxes repo_init @@ -150,23 +150,33 @@ prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +# Explicitly test delta fetches via ref name as well as commit hash +for delta_target in main ${new_rev}; do cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} -${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${delta_target} >dry-run-pull.txt # Compression can vary, so we support 400-699 assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts, 0 bytes/[456][0-9][0-9] bytes, 455 bytes total uncompressed' rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) assert_streq "${prev_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck +done +# Explicitly test delta fetches via ref name as well as commit hash +for delta_target in main ${new_rev}; do cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} -${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main -rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) -assert_streq "${new_rev}" "${rev}" +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${delta_target} +if test ${delta_target} = main; then + rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) + assert_streq "${new_rev}" "${rev}" +else + ${CMD_PREFIX} ostree --repo=repo rev-parse ${delta_target} +fi ${CMD_PREFIX} ostree --repo=repo fsck +done cd ${test_tmpdir} repo_init @@ -208,8 +218,23 @@ fi assert_file_has_content err.txt "deltas required, but none found" ${CMD_PREFIX} ostree --repo=repo fsck +# Now test with a partial commit +repo_init +${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main@${prev_rev} +if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then + assert_not_reached "--require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content err.txt "deltas required, but none found" echo "ok delta required but don't exist" +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${new_rev} 2>err.txt; then + assert_not_reached "--require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content err.txt "deltas required, but none found" +echo "ok delta required for revision" + cd ${test_tmpdir} rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 84320b80..8baee723 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -169,9 +169,9 @@ echo 'ok heuristic endian detection' ${CMD_PREFIX} ostree --repo=repo summary -u mkdir repo2 && ostree_repo_init repo2 --mode=bare-user -${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${newrev} +${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${origrev} ${CMD_PREFIX} ostree --repo=repo2 fsck -${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null +${CMD_PREFIX} ostree --repo=repo2 ls ${origrev} >/dev/null echo 'ok pull delta' From 08964d595dee51655fec7510c0798e8d96e77953 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 18 Apr 2017 10:13:45 -0400 Subject: [PATCH 02/94] checkout: Fix bare-user symlink checkouts Logic error introduced after refactoring; we hoisted the `is_bare_user_symlink` variable to the top, but its computation below. But the `is_bare` symlink depended on it. Closes: https://github.com/ostreedev/ostree/issues/798 Closes: #799 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 2 +- tests/basic-test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 392e16fd..71511824 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -479,7 +479,6 @@ checkout_one_file_at (OstreeRepo *repo, (current_repo->mode == OSTREE_REPO_MODE_BARE && options->mode == OSTREE_REPO_CHECKOUT_MODE_NONE) || (repo_is_usermode && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER); - gboolean is_bare = is_hardlinkable && !is_bare_user_symlink; gboolean current_can_cache = (options->enable_uncompressed_cache && current_repo->enable_uncompressed_cache); gboolean is_archive_z2_with_cache = (current_repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 @@ -491,6 +490,7 @@ checkout_one_file_at (OstreeRepo *repo, * as well as write_object(). */ is_bare_user_symlink = (repo_is_usermode && is_symlink); + const gboolean is_bare = is_hardlinkable && !is_bare_user_symlink; /* Verify if no_copy_fallback is set that we can hardlink, with a * special exception for bare-user symlinks. diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 294854bf..f4b2b118 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -44,6 +44,7 @@ validate_checkout_basic() { assert_has_file baz/cow assert_file_has_content baz/cow moo assert_has_file baz/deeper/ohyeah + assert_symlink_has_content somelink nosuchfile ) } From f6f967f8d9391589f8f5b89153febc28f91cb65e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 18 Apr 2017 11:05:19 -0400 Subject: [PATCH 03/94] Bump release for 2017.5 This commit won't actually *be* 2017.5 since due to the way our infrastructure works, we still want to increment git master to 2017.5. See https://github.com/ostreedev/ostree/pull/800 Closes: #800 Approved by: jlebon --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f6512c1f..c8e02930 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too m4_define([year_version], [2017]) -m4_define([release_version], [4]) +m4_define([release_version], [5]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) From 44456c6dc1032619b40717881c3d8ae73ab09e5a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 12 Apr 2017 20:50:53 -0400 Subject: [PATCH 04/94] lib/boot: Convert bootconfig parser to new code style This is a small one. Closes: #790 Approved by: jlebon --- src/libostree/ostree-bootconfig-parser.c | 46 ++++++++---------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/libostree/ostree-bootconfig-parser.c b/src/libostree/ostree-bootconfig-parser.c index 8031334e..a7c473dd 100644 --- a/src/libostree/ostree-bootconfig-parser.c +++ b/src/libostree/ostree-bootconfig-parser.c @@ -79,23 +79,18 @@ ostree_bootconfig_parser_parse_at (OstreeBootconfigParser *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *contents = NULL; - char **lines = NULL; - char **iter = NULL; - g_return_val_if_fail (!self->parsed, FALSE); - contents = glnx_file_get_contents_utf8_at (dfd, path, NULL, cancellable, error); + g_autofree char *contents = glnx_file_get_contents_utf8_at (dfd, path, NULL, cancellable, error); if (!contents) - goto out; + return FALSE; - lines = g_strsplit (contents, "\n", -1); - for (iter = lines; *iter; iter++) + g_auto(GStrv) lines = g_strsplit (contents, "\n", -1); + for (char **iter = lines; *iter; iter++) { const char *line = *iter; char *keyname = ""; - + if (g_ascii_isalpha (*line)) { char **items = NULL; @@ -115,11 +110,8 @@ ostree_bootconfig_parser_parse_at (OstreeBootconfigParser *self, } self->parsed = TRUE; - - ret = TRUE; - out: - g_strfreev (lines); - return ret; + + return TRUE; } gboolean @@ -166,16 +158,10 @@ ostree_bootconfig_parser_write_at (OstreeBootconfigParser *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - GHashTableIter hashiter; - gpointer hashkey, hashvalue; - GString *buf = g_string_new (""); - guint i; - g_autoptr(GHashTable) written_overrides = NULL; + g_autoptr(GString) buf = g_string_new (""); + g_autoptr(GHashTable) written_overrides = g_hash_table_new (g_str_hash, g_str_equal); - written_overrides = g_hash_table_new (g_str_hash, g_str_equal); - - for (i = 0; i < self->lines->len; i++) + for (guint i = 0; i < self->lines->len; i++) { GVariant *linedata = self->lines->pdata[i]; const char *key; @@ -197,6 +183,8 @@ ostree_bootconfig_parser_write_at (OstreeBootconfigParser *self, } } + GHashTableIter hashiter; + gpointer hashkey, hashvalue; g_hash_table_iter_init (&hashiter, self->options); while (g_hash_table_iter_next (&hashiter, &hashkey, &hashvalue)) { @@ -208,15 +196,11 @@ ostree_bootconfig_parser_write_at (OstreeBootconfigParser *self, if (!glnx_file_replace_contents_at (dfd, path, (guint8*)buf->str, buf->len, GLNX_FILE_REPLACE_NODATASYNC, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - if (buf) - g_string_free (buf, TRUE); - return ret; + return TRUE; } - + gboolean ostree_bootconfig_parser_write (OstreeBootconfigParser *self, GFile *output, From d197bfd1331d68a4eac5d1e0dbf7215fb29d8eb3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 13 Apr 2017 11:22:11 -0400 Subject: [PATCH 05/94] sysroot: Continue conversion of some simpler functions to new style This is only about 40%, and mostly simpler functions. It's nice to switch to `g_autoptr(GMatchInfo)` instead of our inline version. I decided to add more usage of `ot_transfer_out_value()`, though it'd be nice to try to have a copy of that in libglnx (or possibly glib). Closes: #791 Approved by: jlebon --- src/libostree/ostree-sysroot.c | 266 ++++++++++----------------------- 1 file changed, 81 insertions(+), 185 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index bdf7376b..b9f6c66e 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -284,47 +284,33 @@ ostree_sysroot_ensure_initialized (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - struct stat stbuf; - if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, "ostree/repo", 0755, cancellable, error)) - goto out; + return FALSE; if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, "ostree/deploy", 0755, cancellable, error)) - goto out; + return FALSE; + struct stat stbuf; if (fstatat (self->sysroot_fd, "ostree/repo/objects", &stbuf, 0) != 0) { if (errno != ENOENT) - { - glnx_set_prefix_error_from_errno (error, "stat %s", "ostree/repo/objects"); - goto out; - } + return glnx_throw_errno_prefix (error, "stat(ostree/repo/objects)"); else { g_autoptr(GFile) repo_dir = g_file_resolve_relative_path (self->path, "ostree/repo"); glnx_unref_object OstreeRepo *repo = ostree_repo_new (repo_dir); if (!ostree_repo_create (repo, OSTREE_REPO_MODE_BARE, cancellable, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; -} - -static void -match_info_cleanup (void *loc) -{ - GMatchInfo **match = (GMatchInfo**)loc; - if (*match) g_match_info_unref (*match); + return TRUE; } gboolean @@ -333,13 +319,9 @@ _ostree_sysroot_parse_deploy_path_name (const char *name, int *out_serial, GError **error) { - gboolean ret = FALSE; - __attribute__((cleanup(match_info_cleanup))) GMatchInfo *match = NULL; - g_autofree char *serial_str = NULL; static gsize regex_initialized; static GRegex *regex; - if (g_once_init_enter (®ex_initialized)) { regex = g_regex_new ("^([0-9a-f]+)\\.([0-9]+)$", 0, 0, NULL); @@ -347,20 +329,14 @@ _ostree_sysroot_parse_deploy_path_name (const char *name, g_once_init_leave (®ex_initialized, 1); } + g_autoptr(GMatchInfo) match = NULL; if (!g_regex_match (regex, name, 0, &match)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid deploy name '%s', expected CHECKSUM.TREESERIAL", name); - goto out; - } + return glnx_throw (error, "Invalid deploy name '%s', expected CHECKSUM.TREESERIAL", name); + g_autofree char *serial_str = g_match_info_fetch (match, 2); *out_csum = g_match_info_fetch (match, 1); - serial_str = g_match_info_fetch (match, 2); *out_serial = (int)g_ascii_strtoll (serial_str, NULL, 10); - - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -370,49 +346,36 @@ _ostree_sysroot_read_current_subbootversion (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - struct stat stbuf; - g_autofree char *ostree_bootdir_name = g_strdup_printf ("ostree/boot.%d", bootversion); - if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; + g_autofree char *ostree_bootdir_name = g_strdup_printf ("ostree/boot.%d", bootversion); + struct stat stbuf; if (fstatat (self->sysroot_fd, ostree_bootdir_name, &stbuf, AT_SYMLINK_NOFOLLOW) != 0) { if (errno == ENOENT) *out_subbootversion = 0; else - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } else { - g_autofree char *current_subbootdir_name = NULL; - - current_subbootdir_name = glnx_readlinkat_malloc (self->sysroot_fd, ostree_bootdir_name, - cancellable, error); + g_autofree char *current_subbootdir_name = + glnx_readlinkat_malloc (self->sysroot_fd, ostree_bootdir_name, + cancellable, error); if (!current_subbootdir_name) - goto out; - + return FALSE; + if (g_str_has_suffix (current_subbootdir_name, ".0")) *out_subbootversion = 0; else if (g_str_has_suffix (current_subbootdir_name, ".1")) *out_subbootversion = 1; else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid target '%s' in %s", - current_subbootdir_name, - ostree_bootdir_name); - goto out; - } + return glnx_throw (error, "Invalid target '%s' in %s", + current_subbootdir_name, ostree_bootdir_name); } - ret = TRUE; - out: - return ret; + return TRUE; } static gint @@ -451,31 +414,30 @@ _ostree_sysroot_read_boot_loader_configs (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - int fd; /* Temporary owned by iterator */ - g_autofree char *entries_path = g_strdup_printf ("boot/loader.%d/entries", bootversion); - g_autoptr(GPtrArray) ret_loader_configs = NULL; - g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; - ret_loader_configs = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + g_autoptr(GPtrArray) ret_loader_configs = + g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - fd = glnx_opendirat_with_errno (self->sysroot_fd, entries_path, TRUE); + g_autofree char *entries_path = g_strdup_printf ("boot/loader.%d/entries", bootversion); + /* Temporary owned by iterator */ + int fd = glnx_opendirat_with_errno (self->sysroot_fd, entries_path, TRUE); if (fd == -1) { - if (errno == ENOENT) - goto done; + if (errno != ENOENT) + return glnx_throw_errno (error); else { - glnx_set_error_from_errno (error); - goto out; + /* Note early return */ + *out_loader_configs = g_steal_pointer (&ret_loader_configs); + return TRUE; } } + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_take_fd (fd, &dfd_iter, error)) - goto out; + return FALSE; while (TRUE) { @@ -483,28 +445,21 @@ _ostree_sysroot_read_boot_loader_configs (OstreeSysroot *self, struct stat stbuf; if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) - goto out; - + return FALSE; if (dent == NULL) break; if (fstatat (dfd_iter.fd, dent->d_name, &stbuf, 0) != 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); if (g_str_has_prefix (dent->d_name, "ostree-") && g_str_has_suffix (dent->d_name, ".conf") && S_ISREG (stbuf.st_mode)) { glnx_unref_object OstreeBootconfigParser *config = ostree_bootconfig_parser_new (); - + if (!ostree_bootconfig_parser_parse_at (config, dfd_iter.fd, dent->d_name, cancellable, error)) - { - g_prefix_error (error, "Parsing %s: ", dent->d_name); - goto out; - } + return g_prefix_error (error, "Parsing %s: ", dent->d_name), FALSE; g_ptr_array_add (ret_loader_configs, g_object_ref (config)); } @@ -512,13 +467,8 @@ _ostree_sysroot_read_boot_loader_configs (OstreeSysroot *self, /* Callers expect us to give them a sorted array */ g_ptr_array_sort (ret_loader_configs, compare_loader_configs_for_sorting); - - done: - if (out_loader_configs) - *out_loader_configs = g_steal_pointer (&ret_loader_configs); - ret = TRUE; - out: - return ret; + ot_transfer_out_value(out_loader_configs, &ret_loader_configs); + return TRUE; } static gboolean @@ -527,49 +477,34 @@ read_current_bootversion (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; int ret_bootversion; struct stat stbuf; if (fstatat (self->sysroot_fd, "boot/loader", &stbuf, AT_SYMLINK_NOFOLLOW) != 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); ret_bootversion = 0; } else { - g_autofree char *target = NULL; - if (!S_ISLNK (stbuf.st_mode)) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not a symbolic link: boot/loader"); - goto out; - } + return glnx_throw (error, "Not a symbolic link: boot/loader"); - target = glnx_readlinkat_malloc (self->sysroot_fd, "boot/loader", cancellable, error); + g_autofree char *target = + glnx_readlinkat_malloc (self->sysroot_fd, "boot/loader", cancellable, error); if (!target) - goto out; + return FALSE; if (g_strcmp0 (target, "loader.0") == 0) ret_bootversion = 0; else if (g_strcmp0 (target, "loader.1") == 0) ret_bootversion = 1; else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid target '%s' in boot/loader", target); - goto out; - } + return glnx_throw (error, "Invalid target '%s' in boot/loader", target); } - ret = TRUE; *out_bootversion = ret_bootversion; - out: - return ret; + return TRUE; } static gboolean @@ -580,42 +515,29 @@ parse_origin (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GKeyFile) ret_origin = NULL; g_autofree char *origin_path = g_strconcat ("../", deployment_name, ".origin", NULL); - struct stat stbuf; - g_autofree char *origin_contents = NULL; + g_autoptr(GKeyFile) ret_origin = g_key_file_new (); - ret_origin = g_key_file_new (); - + struct stat stbuf; if (fstatat (deployment_dfd, origin_path, &stbuf, 0) != 0) { - if (errno == ENOENT) - ; - else - { - glnx_set_error_from_errno (error); - goto out; - } + if (errno != ENOENT) + return glnx_throw_errno (error); } else { - origin_contents = glnx_file_get_contents_utf8_at (deployment_dfd, origin_path, - NULL, cancellable, error); + g_autofree char *origin_contents = + glnx_file_get_contents_utf8_at (deployment_dfd, origin_path, + NULL, cancellable, error); if (!origin_contents) - goto out; + return FALSE; if (!g_key_file_load_from_data (ret_origin, origin_contents, -1, 0, error)) - goto out; + return g_prefix_error (error, "Parsing %s: ", origin_path), FALSE; } - ret = TRUE; - if (out_origin) - *out_origin = g_steal_pointer (&ret_origin); - out: - if (error) - g_prefix_error (error, "Parsing %s: ", origin_path); - return ret; + ot_transfer_out_value(out_origin, &ret_origin); + return TRUE; } static gboolean @@ -626,14 +548,8 @@ parse_bootlink (const char *bootlink, int *out_treebootserial, GError **error) { - gboolean ret = FALSE; - __attribute__((cleanup(match_info_cleanup))) GMatchInfo *match = NULL; - g_autofree char *bootversion_str = NULL; - g_autofree char *treebootserial_str = NULL; - static gsize regex_initialized; static GRegex *regex; - if (g_once_init_enter (®ex_initialized)) { regex = g_regex_new ("^/ostree/boot.([01])/([^/]+)/([^/]+)/([0-9]+)$", 0, 0, NULL); @@ -641,23 +557,17 @@ parse_bootlink (const char *bootlink, g_once_init_leave (®ex_initialized, 1); } + g_autoptr(GMatchInfo) match = NULL; if (!g_regex_match (regex, bootlink, 0, &match)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid ostree= argument '%s', expected ostree=/ostree/boot.BOOTVERSION/OSNAME/BOOTCSUM/TREESERIAL", bootlink); - goto out; - } - - bootversion_str = g_match_info_fetch (match, 1); + return glnx_throw (error, "Invalid ostree= argument '%s', expected ostree=/ostree/boot.BOOTVERSION/OSNAME/BOOTCSUM/TREESERIAL", bootlink); + + g_autofree char *bootversion_str = g_match_info_fetch (match, 1); + g_autofree char *treebootserial_str = g_match_info_fetch (match, 4); *out_entry_bootversion = (int)g_ascii_strtoll (bootversion_str, NULL, 10); *out_osname = g_match_info_fetch (match, 2); *out_bootcsum = g_match_info_fetch (match, 3); - treebootserial_str = g_match_info_fetch (match, 4); *out_treebootserial = (int)g_ascii_strtoll (treebootserial_str, NULL, 10); - - ret = TRUE; - out: - return ret; + return TRUE; } static char * @@ -787,29 +697,19 @@ list_deployments_process_one_boot_entry (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *ostree_arg = NULL; - glnx_unref_object OstreeDeployment *deployment = NULL; - - ostree_arg = get_ostree_kernel_arg_from_config (config); + g_autofree char *ostree_arg = get_ostree_kernel_arg_from_config (config); if (ostree_arg == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No ostree= kernel argument found"); - goto out; - } - + return glnx_throw (error, "No ostree= kernel argument found"); + + glnx_unref_object OstreeDeployment *deployment = NULL; if (!parse_deployment (self, ostree_arg, &deployment, cancellable, error)) - goto out; - + return FALSE; + ostree_deployment_set_bootconfig (deployment, config); g_ptr_array_add (inout_deployments, g_object_ref (deployment)); - - ret = TRUE; - out: - return ret; + return TRUE; } static gint @@ -1066,37 +966,33 @@ _ostree_sysroot_query_bootloader (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean is_active; - glnx_unref_object OstreeBootloader *ret_loader = NULL; - - ret_loader = (OstreeBootloader*)_ostree_bootloader_syslinux_new (sysroot); + glnx_unref_object OstreeBootloader *ret_loader = + (OstreeBootloader*)_ostree_bootloader_syslinux_new (sysroot); if (!_ostree_bootloader_query (ret_loader, &is_active, cancellable, error)) - goto out; + return FALSE; + if (!is_active) { g_object_unref (ret_loader); ret_loader = (OstreeBootloader*)_ostree_bootloader_grub2_new (sysroot); if (!_ostree_bootloader_query (ret_loader, &is_active, cancellable, error)) - goto out; + return FALSE; } if (!is_active) { g_object_unref (ret_loader); ret_loader = (OstreeBootloader*)_ostree_bootloader_uboot_new (sysroot); if (!_ostree_bootloader_query (ret_loader, &is_active, cancellable, error)) - goto out; + return FALSE; } if (!is_active) g_clear_object (&ret_loader); - ret = TRUE; - if (out_bootloader) - *out_bootloader = g_steal_pointer (&ret_loader); - out: - return ret; + ot_transfer_out_value(out_bootloader, &ret_loader); + return TRUE; } char * From 0d8cd2f0770501fb454a00a5d47ad2d53977f65a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 10:26:35 -0400 Subject: [PATCH 06/94] cmdline: Start conversion to new code style This is just a few. I'm tempted to try out the coccinelle patch for this. Closes: #793 Approved by: jlebon --- src/ostree/ot-admin-builtin-undeploy.c | 28 ++++++-------- src/ostree/ot-admin-functions.c | 22 +++-------- src/ostree/ot-main.c | 53 ++++++++------------------ 3 files changed, 32 insertions(+), 71 deletions(-) diff --git a/src/ostree/ot-admin-builtin-undeploy.c b/src/ostree/ot-admin-builtin-undeploy.c index cc86ee43..4bf41d0c 100644 --- a/src/ostree/ot-admin-builtin-undeploy.c +++ b/src/ostree/ot-admin-builtin-undeploy.c @@ -35,7 +35,6 @@ static GOptionEntry options[] = { gboolean ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GOptionContext) context = NULL; glnx_unref_object OstreeSysroot *sysroot = NULL; const char *deploy_index_str; @@ -48,16 +47,16 @@ ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GEr if (!ostree_admin_option_context_parse (context, options, &argc, &argv, OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, &sysroot, cancellable, error)) - goto out; + return FALSE; if (argc < 2) { ot_util_usage_error (context, "INDEX must be specified", error); - goto out; + return FALSE; } if (!ostree_sysroot_load (sysroot, cancellable, error)) - goto out; + return FALSE; current_deployments = ostree_sysroot_get_deployments (sysroot); deploy_index_str = argv[1]; @@ -65,31 +64,26 @@ ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GEr target_deployment = ot_admin_get_indexed_deployment (sysroot, deploy_index, error); if (!target_deployment) - goto out; + return FALSE; if (target_deployment == ostree_sysroot_get_booted_deployment (sysroot)) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, "Cannot undeploy currently booted deployment %i", deploy_index); - goto out; + return FALSE; } - + g_ptr_array_remove_index (current_deployments, deploy_index); if (!ostree_sysroot_write_deployments (sysroot, current_deployments, cancellable, error)) - goto out; + return FALSE; g_print ("Deleted deployment %s.%d\n", ostree_deployment_get_csum (target_deployment), ostree_deployment_get_deployserial (target_deployment)); - - if (!ostree_sysroot_cleanup (sysroot, cancellable, error)) - { - g_prefix_error (error, "Performing final cleanup: "); - goto out; - } - ret = TRUE; - out: - return ret; + if (!ostree_sysroot_cleanup (sysroot, cancellable, error)) + return g_prefix_error (error, "Performing final cleanup: "), FALSE; + + return TRUE; } diff --git a/src/ostree/ot-admin-functions.c b/src/ostree/ot-admin-functions.c index 7ba2207e..672d384a 100644 --- a/src/ostree/ot-admin-functions.c +++ b/src/ostree/ot-admin-functions.c @@ -33,20 +33,11 @@ ot_admin_require_booted_deployment_or_osname (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (sysroot); - if (booted_deployment == NULL && osname == NULL) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not currently booted into an OSTree system and no --os= argument given"); - goto out; - } - - ret = TRUE; - out: - return ret; + return glnx_throw (error, "Not currently booted into an OSTree system and no --os= argument given"); + return TRUE; } /** @@ -141,7 +132,7 @@ ot_admin_sysroot_lock (OstreeSysroot *sysroot, g_source_set_callback (timeout_src, (GSourceFunc)on_sysroot_lock_timeout, &state, NULL); g_source_attach (timeout_src, state.mainctx); g_source_unref (timeout_src); - + on_sysroot_lock_timeout (&state); ostree_sysroot_lock_async (sysroot, NULL, (GAsyncReadyCallback)on_sysroot_lock_acquired, &state); @@ -161,14 +152,11 @@ gboolean ot_admin_execve_reboot (OstreeSysroot *sysroot, GError **error) { g_autoptr(GFile) real_sysroot = g_file_new_for_path ("/"); - + if (g_file_equal (ostree_sysroot_get_path (sysroot), real_sysroot)) { if (execlp ("systemctl", "systemctl", "reboot", NULL) < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno (error); } return TRUE; diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 7eb65602..c00b4918 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -223,7 +223,6 @@ ostree_option_context_parse (GOptionContext *context, GError **error) { glnx_unref_object OstreeRepo *repo = NULL; - gboolean success = FALSE; /* Entries are listed in --help output in the order added. We add the * main entries ourselves so that we can add the --repo entry first. */ @@ -278,7 +277,7 @@ ostree_option_context_parse (GOptionContext *context, { g_propagate_error (error, g_steal_pointer (&local_error)); } - goto out; + return FALSE; } } else if (opt_repo != NULL) @@ -289,17 +288,14 @@ ostree_option_context_parse (GOptionContext *context, if (!(flags & OSTREE_BUILTIN_FLAG_NO_CHECK)) { if (!ostree_repo_open (repo, cancellable, error)) - goto out; + return FALSE; } } if (out_repo) *out_repo = g_steal_pointer (&repo); - success = TRUE; - -out: - return success; + return TRUE; } gboolean @@ -312,22 +308,19 @@ ostree_admin_option_context_parse (GOptionContext *context, GCancellable *cancellable, GError **error) { - g_autoptr(GFile) sysroot_path = NULL; - glnx_unref_object OstreeSysroot *sysroot = NULL; - gboolean success = FALSE; - /* Entries are listed in --help output in the order added. We add the * main entries ourselves so that we can add the --sysroot entry first. */ g_option_context_add_main_entries (context, global_admin_entries, NULL); if (!ostree_option_context_parse (context, main_entries, argc, argv, OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, error)) - goto out; + return FALSE; + g_autoptr(GFile) sysroot_path = NULL; if (opt_sysroot != NULL) sysroot_path = g_file_new_for_path (opt_sysroot); - sysroot = ostree_sysroot_new (sysroot_path); + glnx_unref_object OstreeSysroot *sysroot = ostree_sysroot_new (sysroot_path); if (flags & OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER) { @@ -338,7 +331,7 @@ ostree_admin_option_context_parse (GOptionContext *context, { g_set_error (error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED, "You must be root to perform this command"); - goto out; + return FALSE; } } @@ -350,15 +343,11 @@ ostree_admin_option_context_parse (GOptionContext *context, g_autofree char *deployment_path = NULL; if (!ostree_sysroot_load (sysroot, cancellable, error)) - goto out; + return FALSE; deployments = ostree_sysroot_get_deployments (sysroot); if (deployments->len == 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unable to find a deployment in sysroot"); - goto out; - } + return glnx_throw (error, "Unable to find a deployment in sysroot"); first_deployment = deployments->pdata[0]; deployment_file = ostree_sysroot_get_deployment_directory (sysroot, first_deployment); deployment_path = g_file_get_path (deployment_file); @@ -379,29 +368,22 @@ ostree_admin_option_context_parse (GOptionContext *context, { /* Released when sysroot is finalized, or on process exit */ if (!ot_admin_sysroot_lock (sysroot, error)) - goto out; + return FALSE; } if (out_sysroot) *out_sysroot = g_steal_pointer (&sysroot); - success = TRUE; - -out: - return success; + return TRUE; } gboolean ostree_ensure_repo_writable (OstreeRepo *repo, GError **error) { - gboolean ret; - - ret = ostree_repo_is_writable (repo, error); - - g_prefix_error (error, "Cannot write to repository: "); - - return ret; + if (!ostree_repo_is_writable (repo, error)) + return g_prefix_error (error, "Cannot write to repository: "), FALSE; + return TRUE; } void @@ -432,7 +414,6 @@ ostree_print_gpg_verify_result (OstreeGpgVerifyResult *result) gboolean ot_enable_tombstone_commits (OstreeRepo *repo, GError **error) { - gboolean ret = FALSE; gboolean tombstone_commits = FALSE; GKeyFile *config = ostree_repo_get_config (repo); @@ -442,10 +423,8 @@ ot_enable_tombstone_commits (OstreeRepo *repo, GError **error) { g_key_file_set_boolean (config, "core", "tombstone-commits", TRUE); if (!ostree_repo_write_config (repo, config, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } From 49a525f6a5b4f99a85bbe653684afd111fa92070 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 15:15:23 -0400 Subject: [PATCH 07/94] repo: Optimize bare-user content object reads a bit `perf record ostree checkout ...` for a bare-user repo was telling me we were spending a good 13% of our time in the depchain of `ot_lgexattrat()`. The problem here is that traversing the `/proc` path turns out to be somewhat expensive - there are LSM (SELinux) checks, etc. For regular files, opening and just getting the xattr, then closing is still quite cheap. For symlinks, we'll always need to open anyways. This appears to shave about ~0.1 seconds off of a checkout of `fedora-atomic/25/x86_64/docker-host` on my workstation. Oh, and this was the last user of `ot_lgexattrat()` so we can kill it, which is nice - the xattr code should really live in libglnx. Closes: #796 Approved by: jlebon --- src/libostree/ostree-repo.c | 29 +++++++--------- src/libotutil/ot-fs-utils.c | 66 ------------------------------------- src/libotutil/ot-fs-utils.h | 13 -------- 3 files changed, 12 insertions(+), 96 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index c8a12543..bfb9fb37 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2770,8 +2770,18 @@ ostree_repo_load_file (OstreeRepo *self, g_autoptr(GBytes) bytes = NULL; glnx_fd_close int fd = -1; - bytes = ot_lgetxattrat (self->objects_dir_fd, loose_path_buf, - "user.ostreemeta", error); + /* In bare-user, symlinks are stored as regular files, so we just + * always do an open, then query the user.ostreemeta xattr for + * more information. + */ + fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + bytes = glnx_fgetxattr_bytes (fd, "user.ostreemeta", error); if (bytes == NULL) goto out; @@ -2783,21 +2793,6 @@ ostree_repo_load_file (OstreeRepo *self, mode = g_file_info_get_attribute_uint32 (ret_file_info, "unix::mode"); - /* Optimize this so that we only open the file if we - * need to; symlinks contain their content, and we only - * open regular files if the caller has requested an - * input stream. - */ - if (S_ISLNK (mode) || out_input) - { - fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); - if (fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } - } - if (S_ISREG (mode) && out_input) { g_assert (fd != -1); diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index a0a72ca5..4ecf820b 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -61,72 +61,6 @@ ot_gopendirat (int dfd, return TRUE; } -GBytes * -ot_lgetxattrat (int dfd, - const char *path, - const char *attribute, - GError **error) -{ - /* A workaround for the lack of lgetxattrat(), thanks to Florian Weimer: - * https://mail.gnome.org/archives/ostree-list/2014-February/msg00017.html - */ - g_autofree char *full_path = g_strdup_printf ("/proc/self/fd/%d/%s", dfd, path); - GBytes *bytes = NULL; - ssize_t bytes_read, real_size; - char *buf; - - do - bytes_read = lgetxattr (full_path, attribute, NULL, 0); - while (G_UNLIKELY (bytes_read < 0 && errno == EINTR)); - if (G_UNLIKELY (bytes_read < 0)) - { - glnx_set_error_from_errno (error); - goto out; - } - - buf = g_malloc (bytes_read); - do - real_size = lgetxattr (full_path, attribute, buf, bytes_read); - while (G_UNLIKELY (real_size < 0 && errno == EINTR)); - if (G_UNLIKELY (real_size < 0)) - { - glnx_set_error_from_errno (error); - g_free (buf); - goto out; - } - - bytes = g_bytes_new_take (buf, real_size); - out: - return bytes; -} - -gboolean -ot_lsetxattrat (int dfd, - const char *path, - const char *attribute, - const void *value, - gsize value_size, - int flags, - GError **error) -{ - /* A workaround for the lack of lsetxattrat(), thanks to Florian Weimer: - * https://mail.gnome.org/archives/ostree-list/2014-February/msg00017.html - */ - g_autofree char *full_path = g_strdup_printf ("/proc/self/fd/%d/%s", dfd, path); - int res; - - do - res = lsetxattr (full_path, "user.ostreemeta", value, value_size, flags); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - return FALSE; - } - - return TRUE; -} - gboolean ot_readlinkat_gfile_info (int dfd, const char *path, diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index 13909c8e..edf8b29a 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -33,19 +33,6 @@ gboolean ot_gopendirat (int dfd, int *out_fd, GError **error); -GBytes * ot_lgetxattrat (int dfd, - const char *path, - const char *attribute, - GError **error); - -gboolean ot_lsetxattrat (int dfd, - const char *path, - const char *attribute, - const void *value, - gsize value_size, - int flags, - GError **error); - gboolean ot_readlinkat_gfile_info (int dfd, const char *path, GFileInfo *target_info, From dea20255313ed6b54d004ef3ba10a8aa04efa9b7 Mon Sep 17 00:00:00 2001 From: Francesco Giannelli Date: Fri, 7 Apr 2017 22:58:55 +0200 Subject: [PATCH 08/94] switchroot: Document a bit more, add demo shell implementation This could help others who want to integrate with other init systems/initramfs. Commit-message-by: Colin Walters Closes: #784 Approved by: cgwalters --- docs/manual/adapting-existing.md | 24 +++++++++++++++--------- src/switchroot/switchroot.sh | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 src/switchroot/switchroot.sh diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md index 275479f0..687f501b 100644 --- a/docs/manual/adapting-existing.md +++ b/docs/manual/adapting-existing.md @@ -71,17 +71,23 @@ directory. ## Booting and initramfs technology -OSTree comes with optional dracut+systemd integration code that parses -the `ostree=` kernel command line argument in the initramfs, and then -sets up the read-only bind mount on `/usr`, a bind mount on the -deployment's `/sysroot` to the physical `/`, and then finally uses -`mount(MS_MOVE)` to make the deployment root appear to be the root -filesystem before telling systemd to switch root. +OSTree comes with optional dracut+systemd integration code which follows +this logic: + +- Parse the `ostree=` kernel command line argument in the initramfs +- Set up a read-only bind mount on `/usr` +- Bind mount the deployment's `/sysroot` to the physical `/` +- Use `mount(MS_MOVE)` to make the deployment root appear to be the root filesystem + +After these steps, systemd switches root. If you are not using dracut or systemd, using OSTree should still be -possible, but you will have to write the integration code. Patches to -support other initramfs technologies and init systems, if sufficiently -clean, will likely be accepted upstream. +possible, but you will have to write the integration code. See the +existing sources in [src/switchroot](/src/switchroot) as a reference, +as well as [src/switchroot/switchroot.sh](/src/switchroot/switchroot.sh). + +Patches to support other initramfs technologies and init systems, if +sufficiently clean, will likely be accepted upstream. A further specific note regarding `sysvinit`: OSTree used to support recording device files such the `/dev/initctl` FIFO, but no longer diff --git a/src/switchroot/switchroot.sh b/src/switchroot/switchroot.sh new file mode 100644 index 00000000..64de80c1 --- /dev/null +++ b/src/switchroot/switchroot.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# This demonstration script is an implementation in shell +# similar to ostree-prepare-root.c. For a bit more information, +# see adapting-existing.md. + +## the ostree boot parameter is avaialbe during the init +env | grep ostree +# ostree=/ostree/boot.1/.../.../0 +## bind mount the ostree deployment to prepare it for move +mount --bind $sysroot$ostree $sysroot$ostree +## bind mount read-only /usr +mount --bind $sysroot$ostree/usr $sysroot$ostree/usr +mount --bind -o remount,ro $sysroot$ostree/usr $sysroot$ostree/usr +## bind mount the physical root +mount --bind $sysroot $sysroot$ostree/sysroot +## bind mount the var directory which is preserved between deployments +mount --bind $sysroot/ostree/deploy/os/var $sysroot$ostree/var +## make sure target directories are present within var +cd $sysroot$ostree/var +mkdir -p roothome mnt opt home +cd - +## move the deployment to the sysroot +mount --move $sysroot$ostree $sysroot +## after these the init system should start the switch root process From 50ca653ff6216ce7b4ccff8eed5c00d62a43e7c2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 13 Apr 2017 21:21:52 -0400 Subject: [PATCH 09/94] repo/checkout: Finish conversion to new code style I plan to make some future changes here, and wanted to do this first. Random side note; how about converting the do/while loops for `EINTR` to `TEMP_FAILURE_RETRY()`? We're very inconsistent about that... Closes: #792 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 337 ++++++++++----------------- 1 file changed, 118 insertions(+), 219 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 71511824..f48aff9e 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -42,64 +42,52 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autofree char *temp_filename = NULL; - g_autoptr(GOutputStream) temp_out = NULL; - glnx_fd_close int fd = -1; - int res; - guint32 file_mode; - /* Don't make setuid files in uncompressed cache */ - file_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode"); + guint32 file_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode"); file_mode &= ~(S_ISUID|S_ISGID); + glnx_fd_close int fd = -1; + g_autofree char *temp_filename = NULL; if (!glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY | O_CLOEXEC, &fd, &temp_filename, error)) - goto out; - temp_out = g_unix_output_stream_new (fd, FALSE); + return FALSE; + g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (fd, FALSE); if (g_output_stream_splice (temp_out, content, 0, cancellable, error) < 0) - goto out; + return FALSE; if (!g_output_stream_flush (temp_out, cancellable, error)) - goto out; + return FALSE; if (!self->disable_fsync) { + int res; do res = fsync (fd); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } if (!g_output_stream_close (temp_out, cancellable, error)) - goto out; + return FALSE; if (fchmod (fd, file_mode) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); if (!_ostree_repo_ensure_loose_objdir_at (self->uncompressed_objects_dir_fd, loose_path, cancellable, error)) - goto out; + return FALSE; if (!glnx_link_tmpfile_at (self->tmp_dir_fd, GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST, fd, temp_filename, self->uncompressed_objects_dir_fd, loose_path, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -119,64 +107,51 @@ write_regular_file_content (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; const OstreeRepoCheckoutMode mode = options->mode; - int fd; - int res; if (g_output_stream_splice (output, input, 0, cancellable, error) < 0) - goto out; + return FALSE; if (!g_output_stream_flush (output, cancellable, error)) - goto out; + return FALSE; - fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)output); + int fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)output); if (mode != OSTREE_REPO_CHECKOUT_MODE_USER) { + int res; do res = fchown (fd, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), g_file_info_get_attribute_uint32 (file_info, "unix::gid")); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); do res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } - + return glnx_throw_errno (error); + if (xattrs) { if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error)) - goto out; + return FALSE; } } - + if (fsync_is_enabled (self, options)) { if (fsync (fd) == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } - - if (!g_output_stream_close (output, cancellable, error)) - goto out; - ret = TRUE; - out: - return ret; + if (!g_output_stream_close (output, cancellable, error)) + return FALSE; + + return TRUE; } static gboolean @@ -190,7 +165,6 @@ checkout_file_from_input_at (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; int res; if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) @@ -203,14 +177,11 @@ checkout_file_from_input_at (OstreeRepo *self, { if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) { - ret = TRUE; - goto out; + /* Note early return */ + return TRUE; } else - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) @@ -219,16 +190,13 @@ checkout_file_from_input_at (OstreeRepo *self, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), g_file_info_get_attribute_uint32 (file_info, "unix::gid"), AT_SYMLINK_NOFOLLOW) == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); if (xattrs) { if (!glnx_dfd_name_set_all_xattrs (destination_dfd, destination_name, xattrs, cancellable, error)) - goto out; + return FALSE; } } } @@ -250,25 +218,23 @@ checkout_file_from_input_at (OstreeRepo *self, { if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) { - ret = TRUE; - goto out; + /* Note early return */ + return TRUE; } - glnx_set_error_from_errno (error); - goto out; + else + return glnx_throw_errno (error); } temp_out = g_unix_output_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ if (!write_regular_file_content (self, options, temp_out, file_info, xattrs, input, cancellable, error)) - goto out; + return FALSE; } else g_assert_not_reached (); - - ret = TRUE; - out: - return ret; + + return TRUE; } /* @@ -286,7 +252,6 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *temp_filename = NULL; if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) @@ -295,20 +260,17 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, g_file_info_get_symlink_target (file_info), &temp_filename, cancellable, error)) - goto out; - + return FALSE; + if (xattrs) { if (!glnx_dfd_name_set_all_xattrs (destination_dfd, temp_filename, xattrs, cancellable, error)) - goto out; + return FALSE; } if (G_UNLIKELY (renameat (destination_dfd, temp_filename, destination_dfd, destination_name) == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { @@ -324,25 +286,23 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, if (!glnx_open_tmpfile_linkable_at (destination_dfd, ".", O_WRONLY | O_CLOEXEC, &temp_fd, &temp_filename, error)) - goto out; + return FALSE; temp_out = g_unix_output_stream_new (temp_fd, FALSE); if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input, cancellable, error)) - goto out; + return FALSE; if (!glnx_link_tmpfile_at (destination_dfd, GLNX_LINK_TMPFILE_REPLACE, temp_fd, temp_filename, destination_dfd, destination_name, error)) - goto out; + return FALSE; } else g_assert_not_reached (); - ret = TRUE; - out: - return ret; + return TRUE; } typedef enum { @@ -386,7 +346,7 @@ checkout_file_hardlink (OstreeRepo *self, ret_result = HARDLINK_RESULT_SKIP_EXISTED; } else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) - { + { /* Idiocy, from man rename(2) * * "If oldpath and newpath are existing hard links referring to @@ -400,9 +360,7 @@ checkout_file_hardlink (OstreeRepo *self, } else { - g_prefix_error (error, "Hardlinking %s to %s: ", loose_path, destination_name); - glnx_set_error_from_errno (error); - return FALSE; + return glnx_throw_errno_prefix (error, "Hardlinking %s to %s", loose_path, destination_name); } if (out_result) @@ -420,22 +378,13 @@ checkout_one_file_at (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const char *checksum; - gboolean is_symlink; - gboolean is_bare_user_symlink = FALSE; - gboolean can_cache; gboolean need_copy = TRUE; + gboolean is_bare_user_symlink = FALSE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - g_autoptr(GInputStream) input = NULL; - g_autoptr(GVariant) xattrs = NULL; - gboolean is_whiteout; - - is_symlink = g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK; - checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source); - - is_whiteout = !is_symlink && options->process_whiteouts && - g_str_has_prefix (destination_name, WHITEOUT_PREFIX); + const gboolean is_symlink = (g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK); + const char *checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source); + const gboolean is_whiteout = (!is_symlink && options->process_whiteouts && + g_str_has_prefix (destination_name, WHITEOUT_PREFIX)); /* First, see if it's a Docker whiteout, * https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go @@ -445,16 +394,12 @@ checkout_one_file_at (OstreeRepo *repo, const char *name = destination_name + (sizeof (WHITEOUT_PREFIX) - 1); if (!name[0]) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid empty whiteout '%s'", name); - goto out; - } + return glnx_throw (error, "Invalid empty whiteout '%s'", name); g_assert (name[0] != '/'); /* Sanity */ if (!glnx_shutil_rm_rf_at (destination_dfd, name, cancellable, error)) - goto out; + return FALSE; need_copy = FALSE; } @@ -496,13 +441,10 @@ checkout_one_file_at (OstreeRepo *repo, * special exception for bare-user symlinks. */ if (options->no_copy_fallback && !is_hardlinkable && !is_bare_user_symlink) - { - glnx_throw (error, - repo_is_usermode ? - "User repository mode requires user checkout mode to hardlink" : - "Bare repository mode cannot hardlink in user checkout mode"); - goto out; - } + return glnx_throw (error, + repo_is_usermode ? + "User repository mode requires user checkout mode to hardlink" : + "Bare repository mode cannot hardlink in user checkout mode"); /* But only under these conditions */ if (is_bare || is_archive_z2_with_cache) @@ -516,24 +458,21 @@ checkout_one_file_at (OstreeRepo *repo, destination_dfd, destination_name, TRUE, &hardlink_res, cancellable, error)) - goto out; + return FALSE; if (hardlink_res == HARDLINK_RESULT_LINKED && options->devino_to_csum_cache) { struct stat stbuf; OstreeDevIno *key; - + if (TEMP_FAILURE_RETRY (fstatat (destination_dfd, destination_name, &stbuf, AT_SYMLINK_NOFOLLOW)) != 0) - { - glnx_set_error_from_errno (error); - goto out; - } - + return glnx_throw_errno (error); + key = g_new (OstreeDevIno, 1); key->dev = stbuf.st_dev; key->ino = stbuf.st_ino; memcpy (key->checksum, checksum, OSTREE_SHA256_STRING_LEN+1); - + g_hash_table_add ((GHashTable*)options->devino_to_csum_cache, key); } @@ -546,8 +485,11 @@ checkout_one_file_at (OstreeRepo *repo, need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } - can_cache = (options->enable_uncompressed_cache - && repo->enable_uncompressed_cache); + const gboolean can_cache = (options->enable_uncompressed_cache + && repo->enable_uncompressed_cache); + + g_autoptr(GInputStream) input = NULL; + g_autoptr(GVariant) xattrs = NULL; /* Ok, if we're archive-z2 and we didn't find an object, uncompress * it now, stick it in the cache, and then hardlink to that. @@ -560,10 +502,10 @@ checkout_one_file_at (OstreeRepo *repo, && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) { HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; - + if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, cancellable, error)) - goto out; + return FALSE; /* Overwrite any parent repo from earlier */ _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, OSTREE_REPO_MODE_BARE); @@ -571,11 +513,8 @@ checkout_one_file_at (OstreeRepo *repo, if (!checkout_object_for_uncompressed_cache (repo, loose_path_buf, source_info, input, cancellable, error)) - { - g_prefix_error (error, "Unpacking loose object %s: ", checksum); - goto out; - } - + return g_prefix_error (error, "Unpacking loose object %s: ", checksum), FALSE; + g_clear_object (&input); /* Store the 2-byte objdir prefix (e.g. e3) in a set. The basic @@ -607,10 +546,7 @@ checkout_one_file_at (OstreeRepo *repo, destination_dfd, destination_name, FALSE, &hardlink_res, cancellable, error)) - { - g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name); - goto out; - } + return g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name), FALSE; need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } @@ -627,18 +563,15 @@ checkout_one_file_at (OstreeRepo *repo, g_assert (is_bare_user_symlink); if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) - goto out; + return FALSE; if (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) { if (!checkout_file_unioning_from_input_at (repo, options, source_info, xattrs, input, destination_dfd, destination_name, - cancellable, error)) - { - g_prefix_error (error, "Union checkout of %s to %s: ", checksum, destination_name); - goto out; - } + cancellable, error)) + return g_prefix_error (error, "Union checkout of %s to %s: ", checksum, destination_name), FALSE; } else { @@ -646,22 +579,17 @@ checkout_one_file_at (OstreeRepo *repo, destination_dfd, destination_name, cancellable, error)) - { - g_prefix_error (error, "Checkout of %s to %s: ", checksum, destination_name); - goto out; - } + return g_prefix_error (error, "Checkout of %s to %s: ", checksum, destination_name), FALSE; } if (input) { if (!g_input_stream_close (input, cancellable, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } /* @@ -689,14 +617,8 @@ checkout_tree_at (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean did_exist = FALSE; - glnx_fd_close int destination_dfd = -1; int res; - struct stat repo_dfd_stat; - struct stat destination_stat; - g_autoptr(GVariant) xattrs = NULL; - g_autoptr(GFileEnumerator) dir_enum = NULL; /* Create initially with mode 0700, then chown/chmod only when we're * done. This avoids anyone else being able to operate on partially @@ -712,65 +634,56 @@ checkout_tree_at (OstreeRepo *self, || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)) did_exist = TRUE; else - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } + glnx_fd_close int destination_dfd = -1; if (!glnx_opendirat (destination_parent_fd, destination_name, TRUE, &destination_dfd, error)) - goto out; + return FALSE; + struct stat repo_dfd_stat; if (fstat (self->repo_dir_fd, &repo_dfd_stat) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); + struct stat destination_stat; if (fstat (destination_dfd, &destination_stat) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); if (options->no_copy_fallback && repo_dfd_stat.st_dev != destination_stat.st_dev) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unable to do hardlink checkout across devices (src=%"G_GUINT64_FORMAT" destination=%"G_GUINT64_FORMAT")", - (guint64)repo_dfd_stat.st_dev, (guint64)destination_stat.st_dev); - goto out; - } + return glnx_throw (error, "Unable to do hardlink checkout across devices (src=%"G_GUINT64_FORMAT" destination=%"G_GUINT64_FORMAT")", + (guint64)repo_dfd_stat.st_dev, (guint64)destination_stat.st_dev); /* Set the xattrs now, so any derived labeling works */ + g_autoptr(GVariant) xattrs = NULL; if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error)) - goto out; + return FALSE; if (xattrs) { if (!glnx_fd_set_all_xattrs (destination_dfd, xattrs, cancellable, error)) - goto out; + return FALSE; } } + /* Note early return here! */ if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) - { - ret = checkout_one_file_at (self, options, - (GFile *) source, - source_info, - destination_dfd, - g_file_info_get_name (source_info), - cancellable, error); - goto out; - } - dir_enum = g_file_enumerate_children ((GFile*)source, - OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); + return checkout_one_file_at (self, options, + (GFile *) source, + source_info, + destination_dfd, + g_file_info_get_name (source_info), + cancellable, error); + + g_autoptr(GFileEnumerator) dir_enum = + g_file_enumerate_children ((GFile*)source, + OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, + error); if (!dir_enum) - goto out; + return FALSE; while (TRUE) { @@ -780,7 +693,7 @@ checkout_tree_at (OstreeRepo *self, if (!g_file_enumerator_iterate (dir_enum, &file_info, &src_child, cancellable, error)) - goto out; + return FALSE; if (file_info == NULL) break; @@ -792,7 +705,7 @@ checkout_tree_at (OstreeRepo *self, destination_dfd, name, (OstreeRepoFile*)src_child, file_info, cancellable, error)) - goto out; + return FALSE; } else { @@ -800,7 +713,7 @@ checkout_tree_at (OstreeRepo *self, src_child, file_info, destination_dfd, name, cancellable, error)) - goto out; + return FALSE; } } @@ -814,10 +727,7 @@ checkout_tree_at (OstreeRepo *self, g_file_info_get_attribute_uint32 (source_info, "unix::mode")); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) @@ -828,10 +738,7 @@ checkout_tree_at (OstreeRepo *self, g_file_info_get_attribute_uint32 (source_info, "unix::gid")); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } /* Set directory mtime to OSTREE_TIMESTAMP, so that it is constant for all checkouts. @@ -844,24 +751,16 @@ checkout_tree_at (OstreeRepo *self, res = futimens (destination_dfd, times); while (G_UNLIKELY (res == -1 && errno == EINTR)); if (G_UNLIKELY (res == -1)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } if (fsync_is_enabled (self, options)) { if (fsync (destination_dfd) == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -1111,7 +1010,7 @@ ostree_repo_checkout_gc (OstreeRepo *self, glnx_set_error_from_errno (error); return FALSE; } - + if (stbuf.st_nlink == 1) { if (unlinkat (dfd_iter.fd, dent->d_name, 0) != 0) From 3f1bcab27f1f6ee77e4a8d2a88858ed2e6d2aa70 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 20 Apr 2017 21:26:17 -0400 Subject: [PATCH 10/94] lib/cleanup: Port some of the cleanup code to fd-relative and new style There aren't many users of `g_file_enumerator_iterate()` left - those remaining are usually good candidates for porting. There's some more porting to do in this file; a mix of trivial and harder. This one is a good candidate for an individual commit. Closes: #803 Approved by: jlebon --- src/libostree/ostree-sysroot-cleanup.c | 171 +++++++++++-------------- src/libostree/ostree-sysroot-deploy.c | 11 +- src/libostree/ostree-sysroot-private.h | 3 +- 3 files changed, 86 insertions(+), 99 deletions(-) diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 022654c3..2933d459 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -25,127 +25,112 @@ #include "ostree-sysroot-private.h" +/* Like glnx_dirfd_iterator_init_at(), but if %ENOENT, then set + * @out_exists to %FALSE, and return successfully. + */ +static gboolean +dfd_iter_init_allow_noent (int dfd, + const char *path, + GLnxDirFdIterator *dfd_iter, + gboolean *out_exists, + GError **error) +{ + glnx_fd_close int fd = glnx_opendirat_with_errno (dfd, path, TRUE); + if (fd < 0) + { + if (errno != ENOENT) + return glnx_throw_errno (error); + *out_exists = FALSE; + return TRUE; + } + if (!glnx_dirfd_iterator_init_take_fd (fd, dfd_iter, error)) + return FALSE; + fd = -1; + *out_exists = TRUE; + return TRUE; +} + +/* @deploydir_dfd: Directory FD for ostree/deploy + * @osname: Target osname + * @inout_deployments: All deployments in this subdir will be appended to this array + */ gboolean -_ostree_sysroot_list_deployment_dirs_for_os (GFile *osdir, +_ostree_sysroot_list_deployment_dirs_for_os (int deploydir_dfd, + const char *osname, GPtrArray *inout_deployments, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const char *osname = glnx_basename (gs_file_get_path_cached (osdir)); - g_autoptr(GFileEnumerator) dir_enum = NULL; - g_autoptr(GFile) osdeploy_dir = NULL; - GError *temp_error = NULL; - - osdeploy_dir = g_file_get_child (osdir, "deploy"); - - dir_enum = g_file_enumerate_children (osdeploy_dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, &temp_error); - if (!dir_enum) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - goto done; - } - else - { - g_propagate_error (error, temp_error); - goto out; - } - } + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + gboolean exists; + const char *osdeploy_path = glnx_strjoina (osname, "/deploy"); + if (!dfd_iter_init_allow_noent (deploydir_dfd, osdeploy_path, &dfd_iter, &exists, error)) + return FALSE; + if (!exists) + return TRUE; while (TRUE) { - const char *name; - GFileInfo *file_info = NULL; - GFile *child = NULL; - glnx_unref_object OstreeDeployment *deployment = NULL; - g_autofree char *csum = NULL; - gint deployserial; + struct dirent *dent; - if (!g_file_enumerator_iterate (dir_enum, &file_info, &child, - cancellable, error)) - goto out; - if (file_info == NULL) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) break; - name = g_file_info_get_name (file_info); - - if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY) + if (dent->d_type != DT_DIR) continue; - if (!_ostree_sysroot_parse_deploy_path_name (name, &csum, &deployserial, error)) - goto out; - - deployment = ostree_deployment_new (-1, osname, csum, deployserial, NULL, -1); - g_ptr_array_add (inout_deployments, g_object_ref (deployment)); + g_autofree char *csum = NULL; + gint deployserial; + if (!_ostree_sysroot_parse_deploy_path_name (dent->d_name, &csum, &deployserial, error)) + return FALSE; + + g_ptr_array_add (inout_deployments, ostree_deployment_new (-1, osname, csum, deployserial, NULL, -1)); } - done: - ret = TRUE; - out: - return ret; + return TRUE; } +/* Return in @out_deployments a new array of OstreeDeployment loaded from the + * filesystem state. + */ static gboolean list_all_deployment_directories (OstreeSysroot *self, GPtrArray **out_deployments, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GFileEnumerator) dir_enum = NULL; - g_autoptr(GFile) deploydir = NULL; - g_autoptr(GPtrArray) ret_deployments = NULL; - GError *temp_error = NULL; + g_autoptr(GPtrArray) ret_deployments = + g_ptr_array_new_with_free_func (g_object_unref); - deploydir = g_file_resolve_relative_path (self->path, "ostree/deploy"); - - ret_deployments = g_ptr_array_new_with_free_func (g_object_unref); - - dir_enum = g_file_enumerate_children (deploydir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); - if (!dir_enum) - { - if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&temp_error); - goto done; - } - else - { - g_propagate_error (error, temp_error); - goto out; - } - } + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + gboolean exists; + if (!dfd_iter_init_allow_noent (self->sysroot_fd, "ostree/deploy", &dfd_iter, &exists, error)) + return FALSE; + if (!exists) + return TRUE; while (TRUE) { - GFileInfo *file_info = NULL; - GFile *child = NULL; + struct dirent *dent; - if (!g_file_enumerator_iterate (dir_enum, &file_info, &child, - NULL, error)) - goto out; - if (file_info == NULL) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) break; - if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY) + if (dent->d_type != DT_DIR) continue; - - if (!_ostree_sysroot_list_deployment_dirs_for_os (child, ret_deployments, + + if (!_ostree_sysroot_list_deployment_dirs_for_os (dfd_iter.fd, dent->d_name, + ret_deployments, cancellable, error)) - goto out; + return FALSE; } - - done: - ret = TRUE; + ot_transfer_out_value (out_deployments, &ret_deployments); - out: - return ret; + return TRUE; } static gboolean @@ -154,7 +139,7 @@ parse_bootdir_name (const char *name, char **out_csum) { const char *lastdash; - + if (out_osname) *out_osname = NULL; if (out_csum) @@ -164,7 +149,7 @@ parse_bootdir_name (const char *name, if (!lastdash) return FALSE; - + if (!ostree_validate_checksum_string (lastdash + 1, NULL)) return FALSE; @@ -183,7 +168,6 @@ list_all_boot_directories (OstreeSysroot *self, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) dir_enum = NULL; g_autoptr(GFile) boot_ostree = NULL; g_autoptr(GPtrArray) ret_bootdirs = NULL; GError *temp_error = NULL; @@ -192,9 +176,10 @@ list_all_boot_directories (OstreeSysroot *self, ret_bootdirs = g_ptr_array_new_with_free_func (g_object_unref); - dir_enum = g_file_enumerate_children (boot_ostree, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, &temp_error); + g_autoptr(GFileEnumerator) dir_enum = + g_file_enumerate_children (boot_ostree, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, &temp_error); if (!dir_enum) { if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index ed8c8bca..6cf4b7eb 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1899,19 +1899,20 @@ allocate_deployserial (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - guint i; int new_deployserial = 0; g_autoptr(GPtrArray) tmp_current_deployments = g_ptr_array_new_with_free_func (g_object_unref); - const char *osdir_name = glnx_strjoina ("ostree/deploy/", osname); - g_autoptr(GFile) osdir = g_file_resolve_relative_path (self->path, osdir_name); + glnx_fd_close int deploy_dfd = -1; + if (!glnx_opendirat (self->sysroot_fd, "ostree/deploy", TRUE, &deploy_dfd, error)) + return FALSE; - if (!_ostree_sysroot_list_deployment_dirs_for_os (osdir, tmp_current_deployments, + if (!_ostree_sysroot_list_deployment_dirs_for_os (deploy_dfd, osname, + tmp_current_deployments, cancellable, error)) return FALSE; - for (i = 0; i < tmp_current_deployments->len; i++) + for (guint i = 0; i < tmp_current_deployments->len; i++) { OstreeDeployment *deployment = tmp_current_deployments->pdata[i]; diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 4bc7802b..26cfd363 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -87,7 +87,8 @@ _ostree_sysroot_parse_deploy_path_name (const char *name, GError **error); gboolean -_ostree_sysroot_list_deployment_dirs_for_os (GFile *osdir, +_ostree_sysroot_list_deployment_dirs_for_os (int deploydir_dfd, + const char *osname, GPtrArray *inout_deployments, GCancellable *cancellable, GError **error); From 6060abbb4bd8751cc69a320b4e8e5ff058a1226b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 21 Apr 2017 15:43:17 -0400 Subject: [PATCH 11/94] repo: Add a "force copy" flag to checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is intended to be used for copying `/usr/etc` → `/etc` for deployments. A TODO here is to use `glnx_file_copy_at()` if the repo mode allows it - then we'd use reflinks if available. Closes: #804 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 4 +++- src/libostree/ostree-repo.h | 3 ++- src/ostree/ot-builtin-checkout.c | 10 +++++++++- tests/basic-test.sh | 17 ++++++++++++++--- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index f48aff9e..5b87c7ec 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -403,7 +403,7 @@ checkout_one_file_at (OstreeRepo *repo, need_copy = FALSE; } - else + else if (!options->force_copy) { HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; /* Try to do a hardlink first, if it's a regular file. This also @@ -895,6 +895,8 @@ ostree_repo_checkout_at (OstreeRepo *self, if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY) options->mode = OSTREE_REPO_CHECKOUT_MODE_USER; + g_return_val_if_fail (!(options->force_copy && options->no_copy_fallback), FALSE); + g_autoptr(GFile) commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error); if (!commit_root) return FALSE; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 1664d65d..2e34c21c 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -767,7 +767,8 @@ typedef struct { gboolean enable_fsync; /* Deprecated */ gboolean process_whiteouts; gboolean no_copy_fallback; - gboolean unused_bools[7]; + gboolean force_copy; /* Since: 2017.6 */ + gboolean unused_bools[6]; const char *subpath; diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 74e27cfb..9ba804f8 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -42,6 +42,7 @@ static gboolean opt_from_stdin; static char *opt_from_file; static gboolean opt_disable_fsync; static gboolean opt_require_hardlinks; +static gboolean opt_force_copy; static gboolean parse_fsync_cb (const char *option_name, @@ -71,6 +72,7 @@ static GOptionEntry options[] = { { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" }, { "fsync", 0, 0, G_OPTION_ARG_CALLBACK, parse_fsync_cb, "Specify how to invoke fsync()", "POLICY" }, { "require-hardlinks", 'H', 0, G_OPTION_ARG_NONE, &opt_require_hardlinks, "Do not fall back to full copies if hardlinking fails", NULL }, + { "force-copy", 'C', 0, G_OPTION_ARG_NONE, &opt_force_copy, "Never hardlink (but may reflink if available)", NULL }, { NULL } }; @@ -89,7 +91,7 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add) + if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add || opt_force_copy) { OstreeRepoCheckoutAtOptions options = { 0, }; @@ -102,6 +104,11 @@ process_one_checkout (OstreeRepo *repo, "Cannot specify both --union and --union-add"); goto out; } + if (opt_require_hardlinks && opt_force_copy) + { + glnx_throw (error, "Cannot specify both --require-hardlinks and --force-copy"); + goto out; + } else if (opt_union) options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; else if (opt_union_add) @@ -111,6 +118,7 @@ process_one_checkout (OstreeRepo *repo, if (subpath) options.subpath = subpath; options.no_copy_fallback = opt_require_hardlinks; + options.force_copy = opt_force_copy; if (!ostree_repo_checkout_at (repo, &options, AT_FDCWD, destination, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index f4b2b118..6ddf7b2e 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..65" +echo "1..66" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' @@ -28,7 +28,7 @@ echo "ok yaml version" CHECKOUT_U_ARG="" COMMIT_ARGS="" DIFF_ARGS="" -if grep bare-user-only repo/config; then +if grep -q bare-user-only repo/config; then # In bare-user-only repos we can only represent files with uid/gid 0, no # xattrs and canonical permissions, so we need to commit them as such, or # we end up with repos that don't pass fsck @@ -50,11 +50,14 @@ validate_checkout_basic() { $OSTREE checkout test2 checkout-test2 validate_checkout_basic checkout-test2 +if grep -q 'mode=bare$' repo/config; then + assert_not_streq $(stat -c '%h' checkout-test2/firstfile) 1 +fi echo "ok checkout" # Note this tests bare-user *and* bare-user-only rm checkout-test2 -rf -if grep bare-user repo/config; then +if grep -q bare-user repo/config; then $OSTREE checkout -U -H test2 checkout-test2 else $OSTREE checkout -H test2 checkout-test2 @@ -78,6 +81,14 @@ fi fi echo "ok checkout -H" +rm checkout-test2 -rf +$OSTREE checkout -C test2 checkout-test2 +for file in firstfile baz/cow baz/alink; do + assert_streq $(stat -c '%h' checkout-test2/$file) 1 +done + +echo "ok checkout -C" + $OSTREE rev-parse test2 $OSTREE rev-parse 'test2^' $OSTREE rev-parse 'test2^^' 2>/dev/null && fatal "rev-parse test2^^ unexpectedly succeeded!" From 0c4aeff1cbed1c97f3461706f2d360d8c08b5183 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 12 Apr 2017 20:31:30 -0400 Subject: [PATCH 12/94] lib/core: Complete conversion to new code style No surprises here, all quite straightforward. Closes: #789 Approved by: jlebon --- src/libostree/ostree-core.c | 204 ++++++++++++------------------------ 1 file changed, 69 insertions(+), 135 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index bab2ab33..7d2ed3c6 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -579,57 +579,45 @@ ostree_content_stream_parse (gboolean compressed, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; guint32 archive_header_size; guchar dummy[4]; gsize bytes_read; - g_autoptr(GInputStream) ret_input = NULL; - g_autoptr(GFileInfo) ret_file_info = NULL; - g_autoptr(GVariant) ret_xattrs = NULL; - g_autoptr(GVariant) file_header = NULL; - g_autofree guchar *buf = NULL; if (!g_input_stream_read_all (input, &archive_header_size, 4, &bytes_read, cancellable, error)) - goto out; + return FALSE; archive_header_size = GUINT32_FROM_BE (archive_header_size); if (archive_header_size > input_length) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "File header size %u exceeds size %" G_GUINT64_FORMAT, - (guint)archive_header_size, input_length); - goto out; - } - if (archive_header_size == 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "File header size is zero"); - goto out; - } + return glnx_throw (error, "File header size %u exceeds size %" G_GUINT64_FORMAT, + (guint)archive_header_size, input_length); + else if (archive_header_size == 0) + return glnx_throw (error, "File header size is zero"); /* Skip over padding */ if (!g_input_stream_read_all (input, dummy, 4, &bytes_read, cancellable, error)) - goto out; + return FALSE; - buf = g_malloc (archive_header_size); + g_autofree guchar *buf = g_malloc (archive_header_size); if (!g_input_stream_read_all (input, buf, archive_header_size, &bytes_read, cancellable, error)) - goto out; - file_header = g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT, - buf, archive_header_size, trusted, - g_free, buf); + return FALSE; + g_autoptr(GVariant) file_header = + g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT, + buf, archive_header_size, trusted, + g_free, buf); buf = NULL; - + g_autoptr(GFileInfo) ret_file_info = NULL; + g_autoptr(GVariant) ret_xattrs = NULL; if (compressed) { if (!zlib_file_header_parse (file_header, out_file_info ? &ret_file_info : NULL, out_xattrs ? &ret_xattrs : NULL, error)) - goto out; + return FALSE; } else { @@ -637,11 +625,12 @@ ostree_content_stream_parse (gboolean compressed, out_file_info ? &ret_file_info : NULL, out_xattrs ? &ret_xattrs : NULL, error)) - goto out; + return FALSE; if (ret_file_info) g_file_info_set_size (ret_file_info, input_length - archive_header_size - 8); } - + + g_autoptr(GInputStream) ret_input = NULL; if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR && out_input) { @@ -658,12 +647,10 @@ ostree_content_stream_parse (gboolean compressed, ret_input = g_object_ref (input); } - ret = TRUE; ot_transfer_out_value (out_input, &ret_input); ot_transfer_out_value (out_file_info, &ret_file_info); ot_transfer_out_value (out_xattrs, &ret_xattrs); - out: - return ret; + return TRUE; } /** @@ -820,43 +807,39 @@ ostree_checksum_file (GFile *f, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GFileInfo) file_info = NULL; - g_autoptr(GInputStream) in = NULL; - g_autoptr(GVariant) xattrs = NULL; - g_autofree guchar *ret_csum = NULL; - if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - file_info = g_file_query_info (f, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); + g_autoptr(GFileInfo) file_info = + g_file_query_info (f, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); if (!file_info) - goto out; + return FALSE; + g_autoptr(GInputStream) in = NULL; if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { in = (GInputStream*)g_file_read (f, cancellable, error); if (!in) - goto out; + return FALSE; } + g_autoptr(GVariant) xattrs = NULL; if (objtype == OSTREE_OBJECT_TYPE_FILE) { if (!glnx_dfd_name_get_all_xattrs (AT_FDCWD, gs_file_get_path_cached (f), &xattrs, cancellable, error)) - goto out; + return FALSE; } + g_autofree guchar *ret_csum = NULL; if (!ostree_checksum_file_from_input (file_info, xattrs, in, objtype, &ret_csum, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; ot_transfer_out_value(out_csum, &ret_csum); - out: - return ret; + return TRUE; } typedef struct { @@ -1623,27 +1606,21 @@ file_header_parse (GVariant *metadata, GVariant **out_xattrs, GError **error) { - gboolean ret = FALSE; guint32 uid, gid, mode, rdev; const char *symlink_target; - g_autoptr(GFileInfo) ret_file_info = NULL; g_autoptr(GVariant) ret_xattrs = NULL; g_variant_get (metadata, "(uuuu&s@a(ayay))", &uid, &gid, &mode, &rdev, &symlink_target, &ret_xattrs); if (rdev != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted archive file; invalid rdev %u", GUINT32_FROM_BE (rdev)); - goto out; - } + return glnx_throw (error, "Corrupted archive file; invalid rdev %u", GUINT32_FROM_BE (rdev)); uid = GUINT32_FROM_BE (uid); gid = GUINT32_FROM_BE (gid); mode = GUINT32_FROM_BE (mode); - ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); + g_autoptr(GFileInfo) ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); if (S_ISREG (mode)) { @@ -1655,16 +1632,12 @@ file_header_parse (GVariant *metadata, } else { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted archive file; invalid mode %u", mode); - goto out; + return glnx_throw (error, "Corrupted archive file; invalid mode %u", mode); } - ret = TRUE; ot_transfer_out_value(out_file_info, &ret_file_info); ot_transfer_out_value(out_xattrs, &ret_xattrs); - out: - return ret; + return TRUE; } /* @@ -1683,28 +1656,21 @@ zlib_file_header_parse (GVariant *metadata, GVariant **out_xattrs, GError **error) { - gboolean ret = FALSE; guint64 size; guint32 uid, gid, mode, rdev; const char *symlink_target; - g_autoptr(GFileInfo) ret_file_info = NULL; g_autoptr(GVariant) ret_xattrs = NULL; g_variant_get (metadata, "(tuuuu&s@a(ayay))", &size, &uid, &gid, &mode, &rdev, &symlink_target, &ret_xattrs); if (rdev != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted archive file; invalid rdev %u", GUINT32_FROM_BE (rdev)); - goto out; - } + return glnx_throw (error, "Corrupted archive file; invalid rdev %u", GUINT32_FROM_BE (rdev)); uid = GUINT32_FROM_BE (uid); gid = GUINT32_FROM_BE (gid); mode = GUINT32_FROM_BE (mode); - ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); - + g_autoptr(GFileInfo) ret_file_info = _ostree_header_gfile_info_new (mode, uid, gid); g_file_info_set_size (ret_file_info, GUINT64_FROM_BE (size)); if (S_ISREG (mode)) @@ -1717,16 +1683,12 @@ zlib_file_header_parse (GVariant *metadata, } else { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted archive file; invalid mode %u", mode); - goto out; + return glnx_throw (error, "Corrupted archive file; invalid mode %u", mode); } - ret = TRUE; ot_transfer_out_value(out_file_info, &ret_file_info); ot_transfer_out_value(out_xattrs, &ret_xattrs); - out: - return ret; + return TRUE; } /** @@ -1829,34 +1791,30 @@ gboolean ostree_validate_structureof_commit (GVariant *commit, GError **error) { - gboolean ret = FALSE; - g_autoptr(GVariant) parent_csum_v = NULL; - g_autoptr(GVariant) content_csum_v = NULL; - g_autoptr(GVariant) metadata_csum_v = NULL; - gsize n_elts; - if (!validate_variant (commit, OSTREE_COMMIT_GVARIANT_FORMAT, error)) - goto out; + return FALSE; + g_autoptr(GVariant) parent_csum_v = NULL; g_variant_get_child (commit, 1, "@ay", &parent_csum_v); + gsize n_elts; (void) g_variant_get_fixed_array (parent_csum_v, &n_elts, 1); if (n_elts > 0) { if (!ostree_validate_structureof_csum_v (parent_csum_v, error)) - goto out; + return FALSE; } + g_autoptr(GVariant) content_csum_v = NULL; g_variant_get_child (commit, 6, "@ay", &content_csum_v); if (!ostree_validate_structureof_csum_v (content_csum_v, error)) - goto out; + return FALSE; + g_autoptr(GVariant) metadata_csum_v = NULL; g_variant_get_child (commit, 7, "@ay", &metadata_csum_v); if (!ostree_validate_structureof_csum_v (metadata_csum_v, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -1873,14 +1831,13 @@ gboolean ostree_validate_structureof_dirtree (GVariant *dirtree, GError **error) { - gboolean ret = FALSE; const char *filename; g_autoptr(GVariant) content_csum_v = NULL; g_autoptr(GVariant) meta_csum_v = NULL; - GVariantIter *contents_iter = NULL; + g_autoptr(GVariantIter) contents_iter = NULL; if (!validate_variant (dirtree, OSTREE_TREE_GVARIANT_FORMAT, error)) - goto out; + return FALSE; g_variant_get_child (dirtree, 0, "a(say)", &contents_iter); @@ -1888,10 +1845,13 @@ ostree_validate_structureof_dirtree (GVariant *dirtree, &filename, &content_csum_v)) { if (!ot_util_filename_validate (filename, error)) - goto out; + return FALSE; if (!ostree_validate_structureof_csum_v (content_csum_v, error)) - goto out; + return FALSE; } + /* Note we only use autoptr in case we broke out of the loop early; + * g_variant_iter_loop() has special semantics. + */ content_csum_v = NULL; g_variant_iter_free (contents_iter); @@ -1901,68 +1861,49 @@ ostree_validate_structureof_dirtree (GVariant *dirtree, &filename, &content_csum_v, &meta_csum_v)) { if (!ot_util_filename_validate (filename, error)) - goto out; + return FALSE; if (!ostree_validate_structureof_csum_v (content_csum_v, error)) - goto out; + return FALSE; if (!ostree_validate_structureof_csum_v (meta_csum_v, error)) - goto out; + return FALSE; } content_csum_v = NULL; meta_csum_v = NULL; - ret = TRUE; - out: - if (contents_iter) - g_variant_iter_free (contents_iter); - return ret; + return TRUE; } static gboolean validate_stat_mode_perms (guint32 mode, GError **error) { - gboolean ret = FALSE; guint32 otherbits = (~S_IFMT & ~S_IRWXU & ~S_IRWXG & ~S_IRWXO & ~S_ISUID & ~S_ISGID & ~S_ISVTX); if (mode & otherbits) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid mode %u; invalid bits in mode", mode); - goto out; - } + return glnx_throw (error, "Invalid mode %u; invalid bits in mode", mode); - ret = TRUE; - out: - return ret; + return TRUE; } /** * ostree_validate_structureof_file_mode: * @mode: A Unix filesystem mode * @error: Error - * + * * Returns: %TRUE if @mode represents a valid file type and permissions */ gboolean ostree_validate_structureof_file_mode (guint32 mode, GError **error) { - gboolean ret = FALSE; - if (!(S_ISREG (mode) || S_ISLNK (mode))) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid file metadata mode %u; not a valid file type", mode); - goto out; - } + return glnx_throw (error, "Invalid file metadata mode %u; not a valid file type", mode); if (!validate_stat_mode_perms (mode, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -1978,28 +1919,21 @@ gboolean ostree_validate_structureof_dirmeta (GVariant *dirmeta, GError **error) { - gboolean ret = FALSE; guint32 mode; if (!validate_variant (dirmeta, OSTREE_DIRMETA_GVARIANT_FORMAT, error)) - goto out; + return FALSE; g_variant_get_child (dirmeta, 2, "u", &mode); mode = GUINT32_FROM_BE (mode); if (!S_ISDIR (mode)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid directory metadata mode %u; not a directory", mode); - goto out; - } + return glnx_throw (error, "Invalid directory metadata mode %u; not a directory", mode); if (!validate_stat_mode_perms (mode, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } /** From 55603a0c5201278ea12b052abbe84ac856b3b657 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 10:34:38 -0400 Subject: [PATCH 13/94] =?UTF-8?q?Rename=20"osname"=20=E2=86=92=20"stateroo?= =?UTF-8?q?t"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I never really liked the term "osname". I feel "stateroot" is a *lot* clearer, since the osname/stateroot mostly just holds `/var`. Further it avoids the `os` prefix which is already overloaded. Some of the existing docs already talked about "operating system state", which further reinforces this. There's *lot* more things than this which reference the term "osname", but I don't want to change *everything* yet in this patch in case we decide to do something different - this just gets the highlights. Closes: #794 Approved by: jlebon --- docs/manual/atomic-upgrades.md | 2 +- docs/manual/deployment.md | 25 +++++++++++++------------ docs/manual/introduction.md | 2 +- man/ostree-admin-config-diff.xml | 4 ++-- man/ostree-admin-deploy.xml | 2 +- man/ostree-admin-os-init.xml | 12 ++++++++---- man/ostree-admin-switch.xml | 2 +- man/ostree-admin-upgrade.xml | 2 +- man/ostree.xml | 5 +++-- src/libostree/ostree-deployment.c | 6 ++++++ 10 files changed, 37 insertions(+), 25 deletions(-) diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md index a60656a8..341372d0 100644 --- a/docs/manual/atomic-upgrades.md +++ b/docs/manual/atomic-upgrades.md @@ -55,7 +55,7 @@ checking it back out of the repo into a deployment. ## Assembling a new deployment directory Given a commit to deploy, OSTree first allocates a directory for -it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. +it. This is of the form `/boot/loader/entries/ostree-$stateroot-$checksum.$serial.conf`. The `$serial` is normally `0`, but if a given commit is deployed more than once, it will be incremented. This is supported because the previous deployment may have diff --git a/docs/manual/deployment.md b/docs/manual/deployment.md index 51554c4f..76d05701 100644 --- a/docs/manual/deployment.md +++ b/docs/manual/deployment.md @@ -8,23 +8,24 @@ operating systems (accessible via `ostree admin`). The core content of these op are treated as read-only, but they transparently share storage. A deployment is physically located at a path of the form -`/ostree/deploy/$osname/deploy/$checksum`. +`/ostree/deploy/$stateroot/deploy/$checksum`. OSTree is designed to boot directly into exactly one deployment at a time; each deployment is intended to be a target for `chroot()` or equivalent. -### "osname": Group of deployments that share /var +### "stateroot" (AKA "osname"): Group of deployments that share /var -Each deployment is grouped in exactly one "osname". From above, you -can see that an osname is physically represented in the -`/ostree/deploy/$osname` directory. For example, OSTree can allow -parallel installing Debian in `/ostree/deploy/debian` and Red Hat -Enterprise Linux in `/ostree/deploy/rhel` (subject to operating system -support, present released versions of these operating systems may not -support this). +Each deployment is grouped in exactly one "stateroot" (also known as an "osname"); +the former term is preferred. -Each osname has exactly one copy of the traditional Unix `/var`, -stored physically in `/ostree/deploy/$osname/var`. OSTree provides +From above, you can see that an stateroot is physically represented in the +`/ostree/deploy/$stateroot` directory. For example, OSTree can allow parallel +installing Debian in `/ostree/deploy/debian` and Red Hat Enterprise Linux in +`/ostree/deploy/rhel` (subject to operating system support, present released +versions of these operating systems may not support this). + +Each stateroot has exactly one copy of the traditional Unix `/var`, +stored physically in `/ostree/deploy/$stateroot/var`. OSTree provides support tools for `systemd` to create a Linux bind mount that ensures the booted deployment sees the shared copy of `/var`. @@ -81,7 +82,7 @@ files. When a tree is deployed, it will have a configuration file generated of the form -`/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. This +`/boot/loader/entries/ostree-$stateroot-$checksum.$serial.conf`. This configuration file will include a special `ostree=` kernel argument that allows the initramfs to find (and `chroot()` into) the specified deployment. diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md index c88d6c14..6cde8ddd 100644 --- a/docs/manual/introduction.md +++ b/docs/manual/introduction.md @@ -97,7 +97,7 @@ parallel install inside an existing OS or distribution occupying the physical `/` root. On each client machine, there is an OSTree repository stored -in `/ostree/repo`, and a set of "deployments" stored in `/ostree/deploy/$OSNAME/$CHECKSUM`. +in `/ostree/repo`, and a set of "deployments" stored in `/ostree/deploy/$STATEROOT/$CHECKSUM`. Each deployment is primarily composed of a set of hardlinks into the repository. This means each version is deduplicated; an upgrade process only costs disk space proportional to the diff --git a/man/ostree-admin-config-diff.xml b/man/ostree-admin-config-diff.xml index 69209e49..256b02fe 100644 --- a/man/ostree-admin-config-diff.xml +++ b/man/ostree-admin-config-diff.xml @@ -66,10 +66,10 @@ Boston, MA 02111-1307, USA. - ="OSNAME" + ="STATEROOT" - Use a different operating system root than the current one. + Use a different operating system stateroot than the current one. diff --git a/man/ostree-admin-deploy.xml b/man/ostree-admin-deploy.xml index ef96203c..347a4ba9 100644 --- a/man/ostree-admin-deploy.xml +++ b/man/ostree-admin-deploy.xml @@ -66,7 +66,7 @@ Boston, MA 02111-1307, USA. - ="OSNAME" + ="STATEROOT" Use a different operating system root than the current one. diff --git a/man/ostree-admin-os-init.xml b/man/ostree-admin-os-init.xml index 77bd126a..10f02ad6 100644 --- a/man/ostree-admin-os-init.xml +++ b/man/ostree-admin-os-init.xml @@ -49,7 +49,7 @@ Boston, MA 02111-1307, USA. - ostree admin os-init OSNAME + ostree admin os-init STATEROOT @@ -57,15 +57,19 @@ Boston, MA 02111-1307, USA. Description - Initializes an new state for an operating system. Ensures that the core subdirectories of /var (/tmp, /lib, /run, and /lock) exist and initialize the given OSNAME as OSTree root. Each deployment location is comprised of a single shared var and a set of deployments (chroots). + Initializes an new stateroot (AKA "osname") for an operating system. + Ensures that the core subdirectories of /var (/tmp, /lib, /run, and + /lock) exist and initialize the given STATEROOT as OSTree stateroot. + Each deployment location is comprised of a single shared + var and a set of deployments (chroots). Example - $ ostree admin os-init gnome-ostree + $ ostree admin os-init exampleos - ostree/deploy/gnome-ostree initialized as OSTree root + ostree/deploy/exampleos initialized as OSTree root diff --git a/man/ostree-admin-switch.xml b/man/ostree-admin-switch.xml index 8072d6b9..86f9be40 100644 --- a/man/ostree-admin-switch.xml +++ b/man/ostree-admin-switch.xml @@ -66,7 +66,7 @@ Boston, MA 02111-1307, USA. - ="OSNAME" + ="STATEROOT" Use a different operating system root than the current one. diff --git a/man/ostree-admin-upgrade.xml b/man/ostree-admin-upgrade.xml index 7766c66e..51900b85 100644 --- a/man/ostree-admin-upgrade.xml +++ b/man/ostree-admin-upgrade.xml @@ -69,7 +69,7 @@ Boston, MA 02111-1307, USA. - ="OSNAME" + ="STATEROOT" Use a different operating system root than the current one. diff --git a/man/ostree.xml b/man/ostree.xml index e31d58b2..940d81d7 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -65,8 +65,9 @@ Boston, MA 02111-1307, USA. Instead, they parallel install to the new toplevel /ostree directory. Each installed system gets its own - /ostree/deploy/osname - directory. + /ostree/deploy/stateroot + directory. (stateroot is the + newer term for osname). Unlike rpm or diff --git a/src/libostree/ostree-deployment.c b/src/libostree/ostree-deployment.c index 67e896bf..8bdc9b57 100644 --- a/src/libostree/ostree-deployment.c +++ b/src/libostree/ostree-deployment.c @@ -39,6 +39,12 @@ ostree_deployment_get_bootcsum (OstreeDeployment *self) return self->bootcsum; } +/* + * ostree_deployment_get_osname: + * @self: Deployemnt + * + * Returns: The "stateroot" name, also known as an "osname" + */ const char * ostree_deployment_get_osname (OstreeDeployment *self) { From f2e92d81f9c7e30c1cfb39364eb547a85a3d63d9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 24 Apr 2017 21:12:21 -0400 Subject: [PATCH 14/94] lib/util: Delete some leftover pre-libglnx directory opening functions These were migrated into libglnx; port the few callers to use that. Closes: #808 Approved by: jlebon --- src/libostree/ostree-repo.c | 2 +- src/libostree/ostree-sysroot-deploy.c | 4 ++-- src/libotutil/ot-fs-utils.c | 26 -------------------------- src/libotutil/ot-fs-utils.h | 7 ------- 4 files changed, 3 insertions(+), 36 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index bfb9fb37..2df6a292 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2465,7 +2465,7 @@ list_loose_objects (OstreeRepo *self, buf[0] = hexchars[c >> 4]; buf[1] = hexchars[c & 0xF]; buf[2] = '\0'; - dfd = ot_opendirat (self->objects_dir_fd, buf, FALSE); + dfd = glnx_opendirat_with_errno (self->objects_dir_fd, buf, FALSE); if (dfd == -1) { if (errno == ENOENT) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 6cf4b7eb..3a3dd4c9 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -255,7 +255,7 @@ ensure_directory_from_template (int orig_etc_fd, g_assert (path != NULL); g_assert (*path != '/' && *path != '\0'); - if (!ot_gopendirat (modified_etc_fd, path, TRUE, &src_dfd, error)) + if (!glnx_opendirat (modified_etc_fd, path, TRUE, &src_dfd, error)) goto out; /* Create with mode 0700, we'll fchmod/fchown later */ @@ -293,7 +293,7 @@ ensure_directory_from_template (int orig_etc_fd, } } - if (!ot_gopendirat (new_etc_fd, path, TRUE, &target_dfd, error)) + if (!glnx_opendirat (new_etc_fd, path, TRUE, &target_dfd, error)) goto out; if (!dirfd_copy_attributes_and_xattrs (modified_etc_fd, path, src_dfd, target_dfd, diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index 4ecf820b..8ba2cffb 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -35,32 +35,6 @@ ot_fdrel_to_gfile (int dfd, const char *path) return g_file_new_for_path (abspath); } -int -ot_opendirat (int dfd, const char *path, gboolean follow) -{ - int flags = O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY; - if (!follow) - flags |= O_NOFOLLOW; - return openat (dfd, path, flags); -} - -gboolean -ot_gopendirat (int dfd, - const char *path, - gboolean follow, - int *out_fd, - GError **error) -{ - int ret = ot_opendirat (dfd, path, follow); - if (ret == -1) - { - glnx_set_error_from_errno (error); - return FALSE; - } - *out_fd = ret; - return TRUE; -} - gboolean ot_readlinkat_gfile_info (int dfd, const char *path, diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index edf8b29a..c7770682 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -26,13 +26,6 @@ G_BEGIN_DECLS GFile * ot_fdrel_to_gfile (int dfd, const char *path); -int ot_opendirat (int dfd, const char *path, gboolean follow); -gboolean ot_gopendirat (int dfd, - const char *path, - gboolean follow, - int *out_fd, - GError **error); - gboolean ot_readlinkat_gfile_info (int dfd, const char *path, GFileInfo *target_info, From 4fc65b808a59d2e0426c7d6d469a19787bdfd7c7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 13:18:03 -0400 Subject: [PATCH 15/94] repo: Drop unused cache variables leftover from pack files These are leftovers from the packfile code and should have been deleted in commit: 2a0601efc790a0c8f783043f2db682eec9ceffaa I noticed this now since I wanted to add a new type of caching. Closes: #795 Approved by: jlebon --- src/libostree/ostree-repo-private.h | 2 -- src/libostree/ostree-repo.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b1a58d61..fa18947e 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -92,8 +92,6 @@ struct OstreeRepo { OstreeRepoTransactionStats txn_stats; GMutex cache_lock; - GPtrArray *cached_meta_indexes; - GPtrArray *cached_content_indexes; gboolean inited; gboolean writable; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2df6a292..e5dbed25 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -536,8 +536,6 @@ ostree_repo_finalize (GObject *object) if (self->config) g_key_file_free (self->config); g_clear_pointer (&self->txn_refs, g_hash_table_destroy); - g_clear_pointer (&self->cached_meta_indexes, (GDestroyNotify) g_ptr_array_unref); - g_clear_pointer (&self->cached_content_indexes, (GDestroyNotify) g_ptr_array_unref); g_clear_error (&self->writable_error); g_clear_pointer (&self->object_sizes, (GDestroyNotify) g_hash_table_unref); g_mutex_clear (&self->cache_lock); From b7afe91e21143d7abb0adde440683a52712aa246 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 14:00:55 -0400 Subject: [PATCH 16/94] repo/checkout: Cache lookups of dirmeta objects I was reading a strace the other day and noticed we were loading the same `.dirmeta` object many times. Unlike the other object types, `.dirmeta` objects don't accumulate much over time; there are only so many directory metadata types. (Without SELinux involved it'd probably be 5-6 I'd guess offhand). For `fedora-atomic/25/x86_64/docker-host` there are currently 34 `.dirmeta` in the tree. But how many times during a checkout did we load those 34 dirmeta objects? With a quick strace: ``` $ strace -s 2048 -f -o strace.log ostree --repo=repo-build checkout -U fedora-atomic/25/x86_64/docker-host host-test-checkout $ grep dirmeta strace.log | wc -l 7165 ``` After, as you'd expect, we just loaded `34` from disk. We do 6 system calls (`openat+fstat+fstat+read+read+close`) per dirmeta, so we dropped a total of 42780 system calls - which is about 20% of the total system calls made. `perf record` tells me that we're spending ~40 of our time in the kernel during a checkout, so reducing syscall traffic helps. Though most of that appears to be in the VFS and XFS layers for `linkat` (which isn't surprising). So how much did perf improve? Well, on my workstation, I get a lot of fluctuation in timing, sometimes by 30%, so this was well within the noise. But it's well worth speeding up checkout, and I think this optimization will shine more as we improve performance elsewhere. Closes: #795 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 4 ++ src/libostree/ostree-repo-private.h | 21 ++++++++++ src/libostree/ostree-repo.c | 60 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 5b87c7ec..4de6caf9 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -799,6 +799,8 @@ ostree_repo_checkout_tree (OstreeRepo *self, /* Backwards compatibility */ options.enable_uncompressed_cache = TRUE; + g_auto(OstreeRepoMemoryCacheRef) memcache_ref; + _ostree_repo_memory_cache_ref_init (&memcache_ref, self); return checkout_tree_at (self, &options, AT_FDCWD, gs_file_get_path_cached (destination), source, source_info, @@ -916,6 +918,8 @@ ostree_repo_checkout_at (OstreeRepo *self, if (!target_info) return FALSE; + g_auto(OstreeRepoMemoryCacheRef) memcache_ref; + _ostree_repo_memory_cache_ref_init (&memcache_ref, self); if (!checkout_tree_at (self, options, destination_dfd, destination_path, diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index fa18947e..74f032d1 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -92,6 +92,9 @@ struct OstreeRepo { OstreeRepoTransactionStats txn_stats; GMutex cache_lock; + guint dirmeta_cache_refcount; + /* char * checksum → GVariant * for dirmeta objects, used in the checkout path */ + GHashTable *dirmeta_cache; gboolean inited; gboolean writable; @@ -126,6 +129,24 @@ typedef struct { char checksum[OSTREE_SHA256_STRING_LEN+1]; } OstreeDevIno; +/* A MemoryCacheRef is an in-memory cache of objects (currently just DIRMETA). This can + * be used when performing an operation that traverses a repository in someway. Currently, + * the primary use case is ostree_repo_checkout_at() avoiding lots of duplicate dirmeta + * lookups. + */ +typedef struct { + OstreeRepo *repo; +} OstreeRepoMemoryCacheRef; + + +void +_ostree_repo_memory_cache_ref_init (OstreeRepoMemoryCacheRef *state, + OstreeRepo *repo); + +void +_ostree_repo_memory_cache_ref_destroy (OstreeRepoMemoryCacheRef *state); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OstreeRepoMemoryCacheRef, _ostree_repo_memory_cache_ref_destroy); + #define OSTREE_REPO_TMPDIR_STAGING "staging-" #define OSTREE_REPO_TMPDIR_FETCHER "fetcher-" diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e5dbed25..051285d4 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -538,6 +538,7 @@ ostree_repo_finalize (GObject *object) g_clear_pointer (&self->txn_refs, g_hash_table_destroy); g_clear_error (&self->writable_error); g_clear_pointer (&self->object_sizes, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&self->dirmeta_cache, (GDestroyNotify) g_hash_table_unref); g_mutex_clear (&self->cache_lock); g_mutex_clear (&self->txn_stats_lock); @@ -2500,6 +2501,26 @@ load_metadata_internal (OstreeRepo *self, g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE); + /* Special caching for dirmeta objects, since they're commonly referenced many + * times. + */ + const gboolean is_dirmeta_cachable = + (objtype == OSTREE_OBJECT_TYPE_DIR_META && out_variant && !out_stream); + if (is_dirmeta_cachable) + { + GMutex *lock = &self->cache_lock; + g_mutex_lock (lock); + GVariant *cache_hit = NULL; + /* Look it up, if we have a cache */ + if (self->dirmeta_cache) + cache_hit = g_hash_table_lookup (self->dirmeta_cache, sha256); + if (cache_hit) + *out_variant = g_variant_ref (cache_hit); + g_mutex_unlock (lock); + if (cache_hit) + return TRUE; + } + _ostree_loose_path (loose_path_buf, sha256, objtype, self->mode); if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, &fd, @@ -2545,6 +2566,16 @@ load_metadata_internal (OstreeRepo *self, data, TRUE); g_variant_ref_sink (ret_variant); } + + /* Now, let's put it in the cache */ + if (is_dirmeta_cachable) + { + GMutex *lock = &self->cache_lock; + g_mutex_lock (lock); + if (self->dirmeta_cache) + g_hash_table_replace (self->dirmeta_cache, g_strdup (sha256), g_variant_ref (ret_variant)); + g_mutex_unlock (lock); + } } else if (out_stream) { @@ -4932,3 +4963,32 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, out: return ret; } + +/* See ostree-repo-private.h for more information about this */ +void +_ostree_repo_memory_cache_ref_init (OstreeRepoMemoryCacheRef *state, + OstreeRepo *repo) +{ + state->repo = g_object_ref (repo); + GMutex *lock = &repo->cache_lock; + g_mutex_lock (lock); + repo->dirmeta_cache_refcount++; + if (repo->dirmeta_cache == NULL) + repo->dirmeta_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref); + g_mutex_unlock (lock); + +} + +/* See ostree-repo-private.h for more information about this */ +void +_ostree_repo_memory_cache_ref_destroy (OstreeRepoMemoryCacheRef *state) +{ + OstreeRepo *repo = state->repo; + GMutex *lock = &repo->cache_lock; + g_mutex_lock (lock); + repo->dirmeta_cache_refcount--; + if (repo->dirmeta_cache_refcount == 0) + g_clear_pointer (&repo->dirmeta_cache, (GDestroyNotify) g_hash_table_unref); + g_mutex_unlock (lock); + g_object_unref (repo); +} From 511b31cfb51e30543e8692e1ff1ec8ba8f37b5dd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 18 Apr 2017 12:41:40 -0400 Subject: [PATCH 17/94] checkout: Merge union/add logic for copies during checkout We really have an astonishing variety of similar functions which write files and symlinks. I was working on a different PR and the duplication between the union-mode and add-mode/none-mode checkout functions bothered me. I realized that the "handle EEXIST" tri-state maps directly to the `GLnxLinkTmpfileReplaceMode`, so deduping things makes even more sense. Closes: #801 Approved by: jlebon --- Makefile-tests.am | 1 + src/libostree/ostree-repo-checkout.c | 180 +++++++++------------------ tests/test-basic-root.sh | 56 +++++++++ 3 files changed, 114 insertions(+), 123 deletions(-) create mode 100755 tests/test-basic-root.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 8389331d..d2059e3a 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -48,6 +48,7 @@ dist_test_scripts = \ tests/test-basic.sh \ tests/test-basic-user.sh \ tests/test-basic-user-only.sh \ + tests/test-basic-root.sh \ tests/test-pull-subpath.sh \ tests/test-archivez.sh \ tests/test-remote-add.sh \ diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 4de6caf9..24bde4a4 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -154,129 +154,66 @@ write_regular_file_content (OstreeRepo *self, return TRUE; } -static gboolean -checkout_file_from_input_at (OstreeRepo *self, - OstreeRepoCheckoutAtOptions *options, - GFileInfo *file_info, - GVariant *xattrs, - GInputStream *input, - int destination_dfd, - const char *destination_name, - GCancellable *cancellable, - GError **error) -{ - int res; - - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) - { - do - res = symlinkat (g_file_info_get_symlink_target (file_info), - destination_dfd, destination_name); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) - { - if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) - { - /* Note early return */ - return TRUE; - } - else - return glnx_throw_errno (error); - } - - if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) - { - if (G_UNLIKELY (fchownat (destination_dfd, destination_name, - g_file_info_get_attribute_uint32 (file_info, "unix::uid"), - g_file_info_get_attribute_uint32 (file_info, "unix::gid"), - AT_SYMLINK_NOFOLLOW) == -1)) - return glnx_throw_errno (error); - - if (xattrs) - { - if (!glnx_dfd_name_set_all_xattrs (destination_dfd, destination_name, - xattrs, cancellable, error)) - return FALSE; - } - } - } - else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - { - g_autoptr(GOutputStream) temp_out = NULL; - int fd; - guint32 file_mode; - - file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); - /* Don't make setuid files on checkout when we're doing --user */ - if (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) - file_mode &= ~(S_ISUID|S_ISGID); - - do - fd = openat (destination_dfd, destination_name, O_WRONLY | O_CREAT | O_EXCL, file_mode); - while (G_UNLIKELY (fd == -1 && errno == EINTR)); - if (fd == -1) - { - if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) - { - /* Note early return */ - return TRUE; - } - else - return glnx_throw_errno (error); - } - temp_out = g_unix_output_stream_new (fd, TRUE); - fd = -1; /* Transfer ownership */ - - if (!write_regular_file_content (self, options, temp_out, file_info, xattrs, input, - cancellable, error)) - return FALSE; - } - else - g_assert_not_reached (); - - return TRUE; -} - /* - * This function creates a file under a temporary name, then rename()s - * it into place. This implements union-like behavior. + * Create a copy of a file, supporting optional union/add behavior. */ static gboolean -checkout_file_unioning_from_input_at (OstreeRepo *repo, - OstreeRepoCheckoutAtOptions *options, - GFileInfo *file_info, - GVariant *xattrs, - GInputStream *input, - int destination_dfd, - const char *destination_name, - GCancellable *cancellable, - GError **error) +create_file_copy_from_input_at (OstreeRepo *repo, + OstreeRepoCheckoutAtOptions *options, + GFileInfo *file_info, + GVariant *xattrs, + GInputStream *input, + int destination_dfd, + const char *destination_name, + GCancellable *cancellable, + GError **error) { g_autofree char *temp_filename = NULL; + const gboolean union_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + const gboolean add_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) { - if (!_ostree_make_temporary_symlink_at (destination_dfd, - g_file_info_get_symlink_target (file_info), - &temp_filename, - cancellable, error)) - return FALSE; - - if (xattrs) + if (symlinkat (g_file_info_get_symlink_target (file_info), + destination_dfd, destination_name) < 0) { - if (!glnx_dfd_name_set_all_xattrs (destination_dfd, temp_filename, - xattrs, cancellable, error)) + /* Handle union/add behaviors if we get EEXIST */ + if (errno == EEXIST && union_mode) + { + /* Unioning? Let's unlink and try again */ + (void) unlinkat (destination_dfd, destination_name, 0); + if (symlinkat (g_file_info_get_symlink_target (file_info), + destination_dfd, destination_name) < 0) + return glnx_throw_errno (error); + } + else if (errno == EEXIST && add_mode) + /* Note early return - we don't want to set the xattrs below */ + return TRUE; + else + return glnx_throw_errno (error); + } + + /* Process ownership and xattrs now that we made the link */ + if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) + { + if (TEMP_FAILURE_RETRY (fchownat (destination_dfd, destination_name, + g_file_info_get_attribute_uint32 (file_info, "unix::uid"), + g_file_info_get_attribute_uint32 (file_info, "unix::gid"), + AT_SYMLINK_NOFOLLOW)) == -1) + return glnx_throw_errno_prefix (error, "fchownat"); + + if (xattrs != NULL && + !glnx_dfd_name_set_all_xattrs (destination_dfd, destination_name, + xattrs, cancellable, error)) return FALSE; } - if (G_UNLIKELY (renameat (destination_dfd, temp_filename, - destination_dfd, destination_name) == -1)) - return glnx_throw_errno (error); } else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { glnx_fd_close int temp_fd = -1; g_autoptr(GOutputStream) temp_out = NULL; guint32 file_mode; + GLnxLinkTmpfileReplaceMode replace_mode; file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); /* Don't make setuid files on checkout when we're doing --user */ @@ -293,7 +230,15 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, cancellable, error)) return FALSE; - if (!glnx_link_tmpfile_at (destination_dfd, GLNX_LINK_TMPFILE_REPLACE, + /* The add/union/none behaviors map directly to GLnxLinkTmpfileReplaceMode */ + if (add_mode) + replace_mode = GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST; + else if (union_mode) + replace_mode = GLNX_LINK_TMPFILE_REPLACE; + else + replace_mode = GLNX_LINK_TMPFILE_NOREPLACE; + + if (!glnx_link_tmpfile_at (destination_dfd, replace_mode, temp_fd, temp_filename, destination_dfd, destination_name, error)) @@ -565,22 +510,11 @@ checkout_one_file_at (OstreeRepo *repo, cancellable, error)) return FALSE; - if (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) - { - if (!checkout_file_unioning_from_input_at (repo, options, source_info, xattrs, input, - destination_dfd, - destination_name, - cancellable, error)) - return g_prefix_error (error, "Union checkout of %s to %s: ", checksum, destination_name), FALSE; - } - else - { - if (!checkout_file_from_input_at (repo, options, source_info, xattrs, input, - destination_dfd, - destination_name, - cancellable, error)) - return g_prefix_error (error, "Checkout of %s to %s: ", checksum, destination_name), FALSE; - } + if (!create_file_copy_from_input_at (repo, options, source_info, xattrs, input, + destination_dfd, + destination_name, + cancellable, error)) + return g_prefix_error (error, "Copy checkout of %s to %s: ", checksum, destination_name), FALSE; if (input) { diff --git a/tests/test-basic-root.sh b/tests/test-basic-root.sh new file mode 100755 index 00000000..a2fa8809 --- /dev/null +++ b/tests/test-basic-root.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +id=$(id -u) + +if test ${id} != 0; then + skip "continued basic tests must be run as root (possibly in a container)" +fi + +setup_test_repository "bare" + +echo "1..1" + +nextid=$(($id + 1)) + +rm checkout-test2 -rf +$OSTREE checkout test2 checkout-test2 +$OSTREE commit ${COMMIT_ARGS} -b test2 --tree=ref=test2 --owner-uid=$nextid +$OSTREE ls test2 baz/cow > ls.txt +assert_file_has_content ls.txt '-00644 '${nextid}' '${id} +# As bare and running as root (e.g. Docker container), do some ownership tests +# https://github.com/ostreedev/ostree/pull/801 +# Both hardlinks and copies should respect ownership, but we don't have -C yet; +# add it when we do. +for opt in -H; do + rm test2-co -rf + $OSTREE checkout ${opt} test2 test2-co + assert_streq "$(stat -c '%u' test2-co/baz/cow)" ${nextid} + assert_streq "$(stat -c '%u' test2-co/baz/alink)" ${nextid} +done +rm test2-co -rf +# But user mode doesn't +$OSTREE checkout -U test2 test2-co +assert_streq "$(stat -c '%u' test2-co/baz/cow)" ${id} +assert_streq "$(stat -c '%u' test2-co/baz/alink)" ${id} +echo "ok ownership" From 8b4196d8f74ab8d9e758fb55b3f8ec589d025da1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 24 Apr 2017 11:01:20 -0400 Subject: [PATCH 18/94] tests: Factor out a libtest-core.sh This could be shared more easily with e.g. rpm-ostree, but what I'm currently working on is installed, privileged (potentially destructive, i.e. VM) tests that will source this separately from the current `libtest.sh`. That does work installed, but in practice is oriented around unit (uninstalled, unprivileged) tests. Closes: #807 Approved by: jlebon --- tests/libtest-core.sh | 111 ++++++++++++++++++++++++++++++++++++++++++ tests/libtest.sh | 95 ++---------------------------------- 2 files changed, 114 insertions(+), 92 deletions(-) create mode 100644 tests/libtest-core.sh diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh new file mode 100644 index 00000000..ae7f381f --- /dev/null +++ b/tests/libtest-core.sh @@ -0,0 +1,111 @@ +# Core source library for shell script tests +# +# Copyright (C) 2017 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +fatal() { + echo $@ 1>&2; exit 1 +} +# fatal() is shorter to type, but retain this alias +assert_not_reached () { + fatal "$@" +} + +# Some tests look for specific English strings. Use a UTF-8 version +# of the C (POSIX) locale if we have one, or fall back to POSIX +# (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) +if locale -a | grep C.UTF-8 >/dev/null; then + export LC_ALL=C.UTF-8 +else + export LC_ALL=C +fi + +# This should really be the default IMO +export G_DEBUG=fatal-warnings +assert_streq () { + test "$1" = "$2" || fatal "$1 != $2" +} + +assert_str_match () { + if ! echo "$1" | grep -E -q "$2"; then + fatal "$1 does not match regexp $2" + fi +} + +assert_not_streq () { + (! test "$1" = "$2") || fatal "$1 == $2" +} + +assert_has_file () { + test -f "$1" || fatal "Couldn't find '$1'" +} + +assert_has_dir () { + test -d "$1" || fatal "Couldn't find '$1'" +} + +assert_not_has_file () { + if test -f "$1"; then + sed -e 's/^/# /' < "$1" >&2 + fatal "File '$1' exists" + fi +} + +assert_not_file_has_content () { + if grep -q -e "$2" "$1"; then + sed -e 's/^/# /' < "$1" >&2 + fatal "File '$1' incorrectly matches regexp '$2'" + fi +} + +assert_not_has_dir () { + if test -d "$1"; then + fatal "Directory '$1' exists" + fi +} + +assert_file_has_content () { + if ! grep -q -e "$2" "$1"; then + sed -e 's/^/# /' < "$1" >&2 + fatal "File '$1' doesn't match regexp '$2'" + fi +} + +assert_symlink_has_content () { + if ! test -L "$1"; then + echo 1>&2 "File '$1' is not a symbolic link" + exit 1 + fi + if ! readlink "$1" | grep -q -e "$2"; then + sed -e 's/^/# /' < "$1" >&2 + echo 1>&2 "Symbolic link '$1' doesn't match regexp '$2'" + exit 1 + fi +} + +assert_file_empty() { + if test -s "$1"; then + sed -e 's/^/# /' < "$1" >&2 + fatal "File '$1' is not empty" + fi +} + +# Use to skip all of these tests +skip() { + echo "1..0 # SKIP" "$@" + exit 0 +} diff --git a/tests/libtest.sh b/tests/libtest.sh index 58351f81..7939e4c7 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -17,6 +17,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. +dn=$(dirname $0) +. ${dn}/libtest-core.sh + if [ -n "${G_TEST_SRCDIR:-}" ]; then test_srcdir="${G_TEST_SRCDIR}/tests" else @@ -29,25 +32,8 @@ else test_builddir=$(dirname $0) fi -fatal() { - echo $@ 1>&2; exit 1 -} -# fatal() is shorter to type, but retain this alias -assert_not_reached () { - fatal "$@" -} - test_tmpdir=$(pwd) -# Some tests look for specific English strings. Use a UTF-8 version -# of the C (POSIX) locale if we have one, or fall back to POSIX -# (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) -if locale -a | grep C.UTF-8 >/dev/null; then - export LC_ALL=C.UTF-8 -else - export LC_ALL=C -fi - # Sanity check that we're in a tmpdir that has # just .testtmp (created by tap-driver for `make check`, # or nothing at all (as ginstest-runner does) @@ -62,8 +48,6 @@ if ! test -f .testtmp; then touch .testtmp fi -export G_DEBUG=fatal-warnings - # Also, unbreak `tar` inside `make check`...Automake will inject # TAR_OPTIONS: --owner=0 --group=0 --numeric-owner presumably so that # tarballs are predictable, except we don't want this in our tests. @@ -124,74 +108,6 @@ else OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd" fi -assert_streq () { - test "$1" = "$2" || fatal "$1 != $2" -} - -assert_str_match () { - if ! echo "$1" | grep -E -q "$2"; then - fatal "$1 does not match regexp $2" - fi -} - -assert_not_streq () { - (! test "$1" = "$2") || fatal "$1 == $2" -} - -assert_has_file () { - test -f "$1" || fatal "Couldn't find '$1'" -} - -assert_has_dir () { - test -d "$1" || fatal "Couldn't find '$1'" -} - -assert_not_has_file () { - if test -f "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' exists" - fi -} - -assert_not_file_has_content () { - if grep -q -e "$2" "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' incorrectly matches regexp '$2'" - fi -} - -assert_not_has_dir () { - if test -d "$1"; then - fatal "Directory '$1' exists" - fi -} - -assert_file_has_content () { - if ! grep -q -e "$2" "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' doesn't match regexp '$2'" - fi -} - -assert_symlink_has_content () { - if ! test -L "$1"; then - echo 1>&2 "File '$1' is not a symbolic link" - exit 1 - fi - if ! readlink "$1" | grep -q -e "$2"; then - sed -e 's/^/# /' < "$1" >&2 - echo 1>&2 "Symbolic link '$1' doesn't match regexp '$2'" - exit 1 - fi -} - -assert_file_empty() { - if test -s "$1"; then - sed -e 's/^/# /' < "$1" >&2 - fatal "File '$1' is not empty" - fi -} - assert_files_hardlinked() { f1=$(stat -c %i $1) f2=$(stat -c %i $2) @@ -541,11 +457,6 @@ os_repository_new_commit () cd ${test_tmpdir} } -skip() { - echo "1..0 # SKIP" "$@" - exit 0 -} - skip_without_user_xattrs () { touch test-xattrs setfattr -n user.testvalue -v somevalue test-xattrs || \ From 654b0c4877d42b3b15a87114408722e843687ded Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 24 Apr 2017 14:40:06 -0400 Subject: [PATCH 19/94] tests/installed: New installed, privileged tests using Fedora AH Our container-driven tests can't e.g. test SELinux sanely, and have to support being run as root *and* non-root too. Use redhat-ci to provision a VM and run tests directly there. These are installed tests too. Closes: https://github.com/ostreedev/ostree/issues/806 Closes: #807 Approved by: jlebon --- .redhat-ci.yml | 34 ++++++++++++++++++ Makefile-tests.am | 2 +- maint.mk | 2 +- tests/installed/README.md | 2 ++ tests/installed/libinsttest.sh | 48 ++++++++++++++++++++++++++ tests/installed/libtest-core.sh | 1 + tests/installed/run.sh | 9 +++++ tests/installed/test-bare-root.sh | 42 ++++++++++++++++++++++ tests/installed/test-deploy-selinux.sh | 21 +++++++++++ tests/libtest.sh | 2 +- 10 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 tests/installed/README.md create mode 100644 tests/installed/libinsttest.sh create mode 120000 tests/installed/libtest-core.sh create mode 100755 tests/installed/run.sh create mode 100755 tests/installed/test-bare-root.sh create mode 100755 tests/installed/test-deploy-selinux.sh diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 9eaf93ca..e5604040 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -101,3 +101,37 @@ tests: artifacts: - test-suite.log + +--- + +inherit: false +branches: + - master + - auto + - try + +context: f25ah-insttest +required: false + +cluster: + hosts: + - name: vmcheck + distro: fedora/25/atomic + container: + image: projectatomic/ostree-tester + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-gtk-doc + +# Copy the build from the container to the host; ideally down the line +# this is installing an RPM via https://github.com/jlebon/redhat-ci/issues/10 +tests: + - make install DESTDIR=$(pwd)/insttree + - rsync -rl -e 'ssh -o User=root' . vmcheck:ostree/ + - ssh root@vmcheck 'ostree admin unlock && rsync -rlv ./ostree/insttree/usr/ /usr/ && ./ostree/tests/installed/run.sh' + +artifacts: + - test-suite.log diff --git a/Makefile-tests.am b/Makefile-tests.am index d2059e3a..a0eefdb9 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -130,7 +130,7 @@ dist_installed_test_data = tests/archive-test.sh \ tests/pre-endian-deltas-repo-little.tar.xz \ $(NULL) -EXTRA_DIST += tests/libtest.sh +EXTRA_DIST += tests/libtest.sh tests/libtest-core.sh dist_test_extra_scripts = \ tests/bootloader-entries-crosscheck.py \ diff --git a/maint.mk b/maint.mk index 3a6c9a88..ff231c7d 100644 --- a/maint.mk +++ b/maint.mk @@ -1123,7 +1123,7 @@ sc_copyright_check: sc_missing_cmd_prefix: @prohibit='^ostree ' \ halt='found missing ${CMD_PREFIX}' \ - in_vc_files='$tests/.*\.sh$$' \ + in_vc_files='$tests/[^/]*\.sh$$' \ $(_sc_search_regexp) # If tests/help-version exists and seems to be new enough, assume that its diff --git a/tests/installed/README.md b/tests/installed/README.md new file mode 100644 index 00000000..45bf7d93 --- /dev/null +++ b/tests/installed/README.md @@ -0,0 +1,2 @@ +This suite of tests is currently run from redhat-ci; +they're intended to run as root. diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh new file mode 100644 index 00000000..4f72b651 --- /dev/null +++ b/tests/installed/libinsttest.sh @@ -0,0 +1,48 @@ +# Common definitions for installed, privileged tests +# +# Copyright (C) 2017 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +dn=$(dirname $0) +. ${dn}/libtest-core.sh + +# Determine our origin refspec - we'll use this as a test base +rpmostree=$(which rpm-ostree 2>/dev/null) +if test -z "${rpmostree}"; then + skip "no rpm-ostree, at some point point this to raw ostree too" +fi + +# We need to be root +assert_streq $(id -u) 0 + +PYTHON= +for py in /usr/bin/python3 /usr/bin/python; do + if ! test -x ${py}; then continue; fi + export PYTHON=${py} + break +done +if test -z "${PYTHON}"; then + fatal "no python found" +fi + +rpmostree_query_json() { + query=$1 + rpm-ostree status --json | $PYTHON -c 'import json,sys; v=json.load(sys.stdin); print(v'${query}')' +} +host_refspec=$(rpmostree_query_json '["deployments"][0]["origin"]') +host_commit=$(rpmostree_query_json '["deployments"][0]["checksum"]') +host_osname=$(rpmostree_query_json '["deployments"][0]["osname"]') diff --git a/tests/installed/libtest-core.sh b/tests/installed/libtest-core.sh new file mode 120000 index 00000000..d26203e2 --- /dev/null +++ b/tests/installed/libtest-core.sh @@ -0,0 +1 @@ +../libtest-core.sh \ No newline at end of file diff --git a/tests/installed/run.sh b/tests/installed/run.sh new file mode 100755 index 00000000..3c60a6e2 --- /dev/null +++ b/tests/installed/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -xeuo pipefail + +dn=$(dirname $0) +for tn in ${dn}/test-*.sh; do + echo Executing: ${tn} + ${tn} +done diff --git a/tests/installed/test-bare-root.sh b/tests/installed/test-bare-root.sh new file mode 100755 index 00000000..0d384c2c --- /dev/null +++ b/tests/installed/test-bare-root.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Tests of the "raw ostree" functionality using the host's ostree repo as uid 0. + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +echo "1..2" + +cd /ostree/repo/tmp +rm co -rf +rm co-testref -rf +ostree refs --delete testref +ostree checkout -H ${host_refspec} co +victim_symlink=/usr/bin/gtar # Seems likely to stick around +# Copy the link to avoid corrupting it +cp co/${victim_symlink}{,.tmp} +mv co/${victim_symlink}{.tmp,} +# Add another xattr to a symlink and a directory, since otherwise this is unusual +setfattr -n security.biometric -v iris co/${victim_symlink} +setfattr -n security.crunchy -v withketchup co/usr/bin +csum=$(ostree commit -b testref --link-checkout-speedup --tree=dir=co) +ostree fsck +ostree ls -X testref ${victim_symlink} > ls.txt +assert_file_has_content ls.txt 'security.biometric' +ostree ls -X ${host_refspec} ${victim_symlink} > ls.txt +assert_not_file_has_content ls.txt security.biometric +ostree ls -X testref usr/bin > ls.txt +assert_file_has_content ls.txt 'security.crunchy' + +ostree checkout -H testref co-testref +getfattr -n security.biometric co-testref/${victim_symlink} > xattr.txt +assert_file_has_content xattr.txt 'security.biometric="iris"' +getfattr -n security.crunchy co-testref/usr/bin > xattr.txt +assert_file_has_content xattr.txt 'security.crunchy="withketchup"' + +rm co -rf +rm co-testref -rf + +echo "ok xattrs" diff --git a/tests/installed/test-deploy-selinux.sh b/tests/installed/test-deploy-selinux.sh new file mode 100755 index 00000000..c4965f87 --- /dev/null +++ b/tests/installed/test-deploy-selinux.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Verify our /etc merge works with selinux + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +# Create a new deployment +ostree admin deploy --karg-proc-cmdline ${host_refspec} +new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1 + +# A set of files that have a variety of security contexts +for file in fstab passwd exports hostname sysctl.conf; do + current=$(cd /etc && ls -Z ${file}) + new=$(cd ${new_deployment_path}/etc && ls -Z ${file}) + assert_streq "${current}" "${new}" +done + +ostree admin undeploy 0 diff --git a/tests/libtest.sh b/tests/libtest.sh index 7939e4c7..c667bcc2 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -18,7 +18,6 @@ # Boston, MA 02111-1307, USA. dn=$(dirname $0) -. ${dn}/libtest-core.sh if [ -n "${G_TEST_SRCDIR:-}" ]; then test_srcdir="${G_TEST_SRCDIR}/tests" @@ -31,6 +30,7 @@ if [ -n "${G_TEST_BUILDDIR:-}" ]; then else test_builddir=$(dirname $0) fi +. ${test_srcdir}/libtest-core.sh test_tmpdir=$(pwd) From e8efd1c8dcaad8fbd3b05c400972d237406263e7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Apr 2017 13:17:15 -0400 Subject: [PATCH 20/94] checkout: Add SELinux labeling for checkout, use in deploy This is a variant of the efforts in https://github.com/ostreedev/ostree/pull/741 Working on `rpm-ostree livefs`, I realized though I needed to just check out *new* files directly into the live `/etc` (and possibly delete obsolete files). The way the current `/etc` merge works is fundamentally different from that. So my plan currently is to probably do something like: - Compute diff - Check out each *new* file individually (as a copy) - Optionally delete obsolete files Also, a few other things become more important - in the current deploy code, we copy all of the files, then relabel them. But we shouldn't expose to *live* systems the race conditions of doing that, plus we should only relabel files we checked out. By converting the deploy's /etc code to use this, we fix the same TODO item there around atomically having the label set up as we create files. And further, if we kill the `/var` relabeling which I think is unnecessary since Anaconda does it, we could delete large chunks of code there. In the implementation, there are two types of things: regular files, and symlinks. For regular files, in the `O_TMPFILE` case, we have the ability to do *everything* atomically (including SELinux labeling) before linking it into place. So let's just use that. For symlinks, we use `setfscreatecon()`. Closes: #797 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 217 ++++++++++++++++++------ src/libostree/ostree-repo.h | 4 +- src/libostree/ostree-sepolicy-private.h | 1 + src/libostree/ostree-sepolicy.c | 41 +++++ src/libostree/ostree-sysroot-deploy.c | 33 ++-- tests/libtest.sh | 3 + 6 files changed, 232 insertions(+), 67 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 24bde4a4..2fd0b46a 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -29,11 +29,17 @@ #include "otutil.h" #include "ostree-repo-file.h" +#include "ostree-sepolicy-private.h" #include "ostree-core-private.h" #include "ostree-repo-private.h" #define WHITEOUT_PREFIX ".wh." +/* Per-checkout call state/caching */ +typedef struct { + GString *selabel_path_buf; +} CheckoutState; + static gboolean checkout_object_for_uncompressed_cache (OstreeRepo *self, const char *loose_path, @@ -160,6 +166,7 @@ write_regular_file_content (OstreeRepo *self, static gboolean create_file_copy_from_input_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, + CheckoutState *state, GFileInfo *file_info, GVariant *xattrs, GInputStream *input, @@ -171,9 +178,34 @@ create_file_copy_from_input_at (OstreeRepo *repo, g_autofree char *temp_filename = NULL; const gboolean union_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; const gboolean add_mode = options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; + const gboolean sepolicy_enabled = options->sepolicy && !repo->disable_xattrs; + g_autoptr(GVariant) modified_xattrs = NULL; + + /* If we're doing SELinux labeling, prepare it */ + if (sepolicy_enabled) + { + /* If doing sepolicy path-based labeling, we don't want to set the + * security.selinux attr via the generic xattr paths in either the symlink + * or regfile cases, so filter it out. + */ + modified_xattrs = _ostree_filter_selinux_xattr (xattrs); + xattrs = modified_xattrs; + } if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) { + g_auto(OstreeSepolicyFsCreatecon) fscreatecon = { 0, }; + + if (sepolicy_enabled) + { + /* For symlinks, since we don't have O_TMPFILE, we use setfscreatecon() */ + if (!_ostree_sepolicy_preparefscreatecon (&fscreatecon, options->sepolicy, + state->selabel_path_buf->str, + g_file_info_get_attribute_uint32 (file_info, "unix::mode"), + error)) + return FALSE; + } + if (symlinkat (g_file_info_get_symlink_target (file_info), destination_dfd, destination_name) < 0) { @@ -226,6 +258,18 @@ create_file_copy_from_input_at (OstreeRepo *repo, return FALSE; temp_out = g_unix_output_stream_new (temp_fd, FALSE); + if (sepolicy_enabled) + { + g_autofree char *label = NULL; + if (!ostree_sepolicy_get_label (options->sepolicy, + state->selabel_path_buf->str, + g_file_info_get_attribute_uint32 (file_info, "unix::mode"), + &label, cancellable, error)) + return FALSE; + if (fsetxattr (temp_fd, "security.selinux", label, strlen (label), 0) < 0) + return glnx_throw_errno_prefix (error, "Setting security.selinux"); + } + if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input, cancellable, error)) return FALSE; @@ -316,6 +360,7 @@ checkout_file_hardlink (OstreeRepo *self, static gboolean checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, + CheckoutState *state, GFile *source, GFileInfo *source_info, int destination_dfd, @@ -510,9 +555,8 @@ checkout_one_file_at (OstreeRepo *repo, cancellable, error)) return FALSE; - if (!create_file_copy_from_input_at (repo, options, source_info, xattrs, input, - destination_dfd, - destination_name, + if (!create_file_copy_from_input_at (repo, options, state, source_info, xattrs, input, + destination_dfd, destination_name, cancellable, error)) return g_prefix_error (error, "Copy checkout of %s to %s: ", checksum, destination_name), FALSE; @@ -530,6 +574,7 @@ checkout_one_file_at (OstreeRepo *repo, * checkout_tree_at: * @self: Repo * @mode: Options controlling all files + * @state: Any state we're carrying through * @overwrite_mode: Whether or not to overwrite files * @destination_parent_fd: Place tree here * @destination_name: Use this name for tree @@ -542,35 +587,63 @@ checkout_one_file_at (OstreeRepo *repo, * relative @destination_name, located by @destination_parent_fd. */ static gboolean -checkout_tree_at (OstreeRepo *self, - OstreeRepoCheckoutAtOptions *options, - int destination_parent_fd, - const char *destination_name, - OstreeRepoFile *source, - GFileInfo *source_info, - GCancellable *cancellable, - GError **error) +checkout_tree_at_recurse (OstreeRepo *self, + OstreeRepoCheckoutAtOptions *options, + CheckoutState *state, + int destination_parent_fd, + const char *destination_name, + OstreeRepoFile *source, + GFileInfo *source_info, + GCancellable *cancellable, + GError **error) { gboolean did_exist = FALSE; int res; + const gboolean sepolicy_enabled = options->sepolicy && !self->disable_xattrs; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GVariant) modified_xattrs = NULL; - /* Create initially with mode 0700, then chown/chmod only when we're - * done. This avoids anyone else being able to operate on partially - * constructed dirs. - */ - do - res = mkdirat (destination_parent_fd, destination_name, 0700); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) + if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { - if (errno == EEXIST && - (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES - || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)) - did_exist = TRUE; - else - return glnx_throw_errno (error); + if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error)) + return FALSE; } + /* First, make the directory. Push a new scope in case we end up using + * setfscreatecon(). + */ + { + g_auto(OstreeSepolicyFsCreatecon) fscreatecon = { 0, }; + + /* If we're doing SELinux labeling, prepare it */ + if (sepolicy_enabled) + { + /* We'll set the xattr via setfscreatecon(), so don't do it via generic xattrs below. */ + modified_xattrs = _ostree_filter_selinux_xattr (xattrs); + xattrs = modified_xattrs; + + if (!_ostree_sepolicy_preparefscreatecon (&fscreatecon, options->sepolicy, + state->selabel_path_buf->str, + g_file_info_get_attribute_uint32 (source_info, "unix::mode"), + error)) + return FALSE; + } + + /* Create initially with mode 0700, then chown/chmod only when we're + * done. This avoids anyone else being able to operate on partially + * constructed dirs. + */ + if (TEMP_FAILURE_RETRY (mkdirat (destination_parent_fd, destination_name, 0700)) < 0) + { + if (errno == EEXIST && + (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES + || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)) + did_exist = TRUE; + else + return glnx_throw_errno (error); + } + } + glnx_fd_close int destination_dfd = -1; if (!glnx_opendirat (destination_parent_fd, destination_name, TRUE, &destination_dfd, error)) @@ -587,23 +660,16 @@ checkout_tree_at (OstreeRepo *self, return glnx_throw (error, "Unable to do hardlink checkout across devices (src=%"G_GUINT64_FORMAT" destination=%"G_GUINT64_FORMAT")", (guint64)repo_dfd_stat.st_dev, (guint64)destination_stat.st_dev); - /* Set the xattrs now, so any derived labeling works */ - g_autoptr(GVariant) xattrs = NULL; - if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) + /* Set the xattrs if we created the dir */ + if (!did_exist && xattrs) { - if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error)) + if (!glnx_fd_set_all_xattrs (destination_dfd, xattrs, cancellable, error)) return FALSE; - - if (xattrs) - { - if (!glnx_fd_set_all_xattrs (destination_dfd, xattrs, cancellable, error)) - return FALSE; - } } /* Note early return here! */ if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) - return checkout_one_file_at (self, options, + return checkout_one_file_at (self, options, state, (GFile *) source, source_info, destination_dfd, @@ -624,6 +690,7 @@ checkout_tree_at (OstreeRepo *self, GFileInfo *file_info; GFile *src_child; const char *name; + GString *selabel_path_buf = state->selabel_path_buf; if (!g_file_enumerator_iterate (dir_enum, &file_info, &src_child, cancellable, error)) @@ -632,23 +699,33 @@ checkout_tree_at (OstreeRepo *self, break; name = g_file_info_get_name (file_info); + size_t namelen = strlen (name); + if (selabel_path_buf) + g_string_append_len (selabel_path_buf, name, namelen); if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) { - if (!checkout_tree_at (self, options, - destination_dfd, name, - (OstreeRepoFile*)src_child, file_info, - cancellable, error)) + if (selabel_path_buf) + g_string_append_c (selabel_path_buf, '/'); + if (!checkout_tree_at_recurse (self, options, state, + destination_dfd, name, + (OstreeRepoFile*)src_child, file_info, + cancellable, error)) return FALSE; + if (state->selabel_path_buf) + g_string_truncate (selabel_path_buf, state->selabel_path_buf->len-1); } else { - if (!checkout_one_file_at (self, options, + if (!checkout_one_file_at (self, options, state, src_child, file_info, destination_dfd, name, cancellable, error)) return FALSE; } + + if (selabel_path_buf) + g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); } /* We do fchmod/fchown last so that no one else could access the @@ -697,6 +774,51 @@ checkout_tree_at (OstreeRepo *self, return TRUE; } +/* Begin a checkout process */ +static gboolean +checkout_tree_at (OstreeRepo *self, + OstreeRepoCheckoutAtOptions *options, + int destination_parent_fd, + const char *destination_name, + OstreeRepoFile *source, + GFileInfo *source_info, + GCancellable *cancellable, + GError **error) +{ + CheckoutState state = { 0, }; + // If SELinux labeling is enabled, we need to keep track of the full path string + if (options->sepolicy) + { + GString *buf = g_string_new (options->sepolicy_prefix ?: options->subpath); + g_assert_cmpint (buf->len, >, 0); + // Ensure it ends with / + if (buf->str[buf->len-1] != '/') + g_string_append_c (buf, '/'); + state.selabel_path_buf = buf; + + /* Otherwise it'd just be corrupting things, and there's no use case */ + g_assert (options->force_copy); + } + + return checkout_tree_at_recurse (self, options, &state, destination_parent_fd, + destination_name, + source, source_info, + cancellable, error); +} + +static void +canonicalize_options (OstreeRepo *self, + OstreeRepoCheckoutAtOptions *options) +{ + /* Canonicalize subpath to / */ + if (!options->subpath) + options->subpath = "/"; + + /* Force USER mode for BARE_USER_ONLY always - nothing else makes sense */ + if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY) + options->mode = OSTREE_REPO_CHECKOUT_MODE_USER; +} + /** * ostree_repo_checkout_tree: * @self: Repo @@ -724,14 +846,11 @@ ostree_repo_checkout_tree (OstreeRepo *self, GError **error) { OstreeRepoCheckoutAtOptions options = { 0, }; - - if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY) - mode = OSTREE_REPO_CHECKOUT_MODE_USER; - options.mode = mode; options.overwrite_mode = overwrite_mode; /* Backwards compatibility */ options.enable_uncompressed_cache = TRUE; + canonicalize_options (self, &options); g_auto(OstreeRepoMemoryCacheRef) memcache_ref; _ostree_repo_memory_cache_ref_init (&memcache_ref, self); @@ -827,11 +946,10 @@ ostree_repo_checkout_at (OstreeRepo *self, /* Make a copy so we can modify the options */ real_options = *options; options = &real_options; - - if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY) - options->mode = OSTREE_REPO_CHECKOUT_MODE_USER; + canonicalize_options (self, options); g_return_val_if_fail (!(options->force_copy && options->no_copy_fallback), FALSE); + g_return_val_if_fail (!options->sepolicy || options->force_copy, FALSE); g_autoptr(GFile) commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error); if (!commit_root) @@ -841,7 +959,8 @@ ostree_repo_checkout_at (OstreeRepo *self, return FALSE; g_autoptr(GFile) target_dir = NULL; - if (options->subpath && strcmp (options->subpath, "/") != 0) + + if (strcmp (options->subpath, "/") != 0) target_dir = g_file_get_child (commit_root, options->subpath); else target_dir = g_object_ref (commit_root); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 2e34c21c..86bed09c 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -775,7 +775,9 @@ typedef struct { OstreeRepoDevInoCache *devino_to_csum_cache; int unused_ints[6]; - gpointer unused_ptrs[7]; + gpointer unused_ptrs[5]; + OstreeSePolicy *sepolicy; /* Since: 2017.6 */ + const char *sepolicy_prefix; } OstreeRepoCheckoutAtOptions; _OSTREE_PUBLIC diff --git a/src/libostree/ostree-sepolicy-private.h b/src/libostree/ostree-sepolicy-private.h index 55d49eaf..def8ab74 100644 --- a/src/libostree/ostree-sepolicy-private.h +++ b/src/libostree/ostree-sepolicy-private.h @@ -37,5 +37,6 @@ gboolean _ostree_sepolicy_preparefscreatecon (OstreeSepolicyFsCreatecon *con, guint32 mode, GError **error); +GVariant *_ostree_filter_selinux_xattr (GVariant *xattrs); G_END_DECLS diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index 6063022c..84b2d5c7 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -730,3 +730,44 @@ _ostree_sepolicy_fscreatecon_clear (OstreeSepolicyFsCreatecon *con) return; ostree_sepolicy_fscreatecon_cleanup (NULL); } + +/* + * Given @xattrs, filter out `security.selinux`, and return + * a new GVariant without it. Supports @xattrs as %NULL to + * mean "no xattrs", and also returns %NULL if no xattrs + * would result (rather than a zero-length array). + */ +GVariant * +_ostree_filter_selinux_xattr (GVariant *xattrs) +{ + if (!xattrs) + return NULL; + + gboolean have_xattrs = FALSE; + GVariantBuilder builder; + guint n = g_variant_n_children (xattrs); + for (guint i = 0; i < n; i++) + { + const char *name = NULL; + g_autoptr(GVariant) value = NULL; + + g_variant_get_child (xattrs, i, "(^&ay@ay)", + &name, &value); + + if (strcmp (name, "security.selinux") == 0) + continue; + /* Initialize builder lazily */ + if (!have_xattrs) + { + have_xattrs = TRUE; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a(ayay)")); + } + g_variant_builder_add (&builder, "(@ay@ay)", + g_variant_new_bytestring (name), + value); + } + /* Canonicalize zero length to NULL for efficiency */ + if (!have_xattrs) + return NULL; + return g_variant_ref_sink (g_variant_builder_end (&builder)); +} diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 3a3dd4c9..57020fa8 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -767,6 +767,7 @@ selinux_relabel_var_if_needed (OstreeSysroot *sysroot, static gboolean merge_configuration (OstreeSysroot *sysroot, + OstreeRepo *repo, OstreeDeployment *previous_deployment, OstreeDeployment *deployment, int deployment_dfd, @@ -829,19 +830,15 @@ merge_configuration (OstreeSysroot *sysroot, usretc_exists = TRUE; etc_exists = FALSE; } - + if (usretc_exists) { - glnx_fd_close int deployment_usr_dfd = -1; - - if (!glnx_opendirat (deployment_dfd, "usr", TRUE, &deployment_usr_dfd, error)) - goto out; - - /* TODO - set out labels as we copy files */ - g_assert (!etc_exists); - if (!copy_dir_recurse (deployment_usr_dfd, deployment_dfd, "etc", - sysroot->debug_flags, cancellable, error)) - goto out; + /* We need copies of /etc from /usr/etc (so admins can use vi), and if + * SELinux is enabled, we need to relabel. + */ + OstreeRepoCheckoutAtOptions etc_co_opts = { .force_copy = TRUE, + .subpath = "/usr/etc", + .sepolicy_prefix = "/etc"}; /* Here, we initialize SELinux policy from the /usr/etc inside * the root - this is before we've finalized the configuration @@ -849,13 +846,15 @@ merge_configuration (OstreeSysroot *sysroot, sepolicy = ostree_sepolicy_new (deployment_path, cancellable, error); if (!sepolicy) goto out; - if (ostree_sepolicy_get_name (sepolicy) != NULL) - { - if (!selinux_relabel_dir (sysroot, sepolicy, deployment_etc_path, "etc", + etc_co_opts.sepolicy = sepolicy; + + /* Copy usr/etc → etc */ + if (!ostree_repo_checkout_at (repo, &etc_co_opts, + deployment_dfd, "etc", + ostree_deployment_get_csum (deployment), cancellable, error)) - goto out; - } + goto out; } if (source_etc_path) @@ -2018,7 +2017,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, ostree_deployment_set_bootconfig (new_deployment, bootconfig); glnx_unref_object OstreeSePolicy *sepolicy = NULL; - if (!merge_configuration (self, merge_deployment, new_deployment, + if (!merge_configuration (self, repo, merge_deployment, new_deployment, deployment_dfd, &sepolicy, cancellable, error)) diff --git a/tests/libtest.sh b/tests/libtest.sh index c667bcc2..8127982d 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -407,6 +407,9 @@ EOF mkdir sysroot export OSTREE_SYSROOT=sysroot ${CMD_PREFIX} ostree admin init-fs sysroot + if test -n "${OSTREE_NO_XATTRS:-}"; then + echo -e 'disable-xattrs=true\n' >> sysroot/ostree/repo/config + fi ${CMD_PREFIX} ostree admin os-init testos case $bootmode in From 8d58ab1002cbc4a1ecafe3d1a80984f8a60f41e9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 25 Apr 2017 11:37:16 -0400 Subject: [PATCH 21/94] repo: Port object listing func to use libglnx more + new style This did a `closedir` in the `goto out` section before, but it turns out more nicely if we follow the usual pattern of doing the `open(O_DIRECTORY)` in the callee function and handle `ENOENT` there. Closes: #809 Approved by: jlebon --- src/libostree/ostree-repo.c | 61 ++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 051285d4..2788d330 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2364,39 +2364,45 @@ ostree_repo_get_parent (OstreeRepo *self) static gboolean list_loose_objects_at (OstreeRepo *self, GHashTable *inout_objects, - const char *prefix, int dfd, + const char *prefix, const char *commit_starting_with, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - DIR *d = NULL; - struct dirent *dent; GVariant *key, *value; - d = fdopendir (dfd); - if (!d) + glnx_fd_close int target_dfd = glnx_opendirat_with_errno (dfd, prefix, FALSE); + if (target_dfd < 0) { - glnx_set_error_from_errno (error); - goto out; + /* Nothing to do if this dir doesn't exist */ + if (errno == ENOENT) + return TRUE; + return glnx_throw_errno (error); } + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + if (!glnx_dirfd_iterator_init_take_fd (target_dfd, &dfd_iter, error)) + return FALSE; - while ((dent = readdir (d)) != NULL) + while (TRUE) { - const char *name = dent->d_name; - const char *dot; - OstreeObjectType objtype; - char buf[OSTREE_SHA256_STRING_LEN+1]; + struct dirent *dent; + if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + + const char *name = dent->d_name; if (strcmp (name, ".") == 0 || strcmp (name, "..") == 0) continue; - dot = strrchr (name, '.'); + const char *dot = strrchr (name, '.'); if (!dot) continue; + OstreeObjectType objtype; if ((self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && strcmp (dot, ".filez") == 0) || ((_ostree_repo_mode_is_bare (self->mode)) @@ -2414,6 +2420,8 @@ list_loose_objects_at (OstreeRepo *self, if ((dot - name) != 62) continue; + char buf[OSTREE_SHA256_STRING_LEN+1]; + memcpy (buf, prefix, 2); memcpy (buf + 2, name, 62); buf[sizeof(buf)-1] = '\0'; @@ -2440,11 +2448,7 @@ list_loose_objects_at (OstreeRepo *self, g_variant_ref_sink (value)); } - ret = TRUE; - out: - if (d) - (void) closedir (d); - return ret; + return TRUE; } static gboolean @@ -2454,28 +2458,17 @@ list_loose_objects (OstreeRepo *self, GCancellable *cancellable, GError **error) { - guint c; - int dfd = -1; static const gchar hexchars[] = "0123456789abcdef"; - for (c = 0; c < 256; c++) + for (guint c = 0; c < 256; c++) { char buf[3]; buf[0] = hexchars[c >> 4]; buf[1] = hexchars[c & 0xF]; buf[2] = '\0'; - dfd = glnx_opendirat_with_errno (self->objects_dir_fd, buf, FALSE); - if (dfd == -1) - { - if (errno == ENOENT) - continue; - else - return glnx_throw_errno (error); - } - /* Takes ownership of dfd */ - if (!list_loose_objects_at (self, inout_objects, buf, dfd, - commit_starting_with, - cancellable, error)) + if (!list_loose_objects_at (self, inout_objects, self->objects_dir_fd, buf, + commit_starting_with, + cancellable, error)) return FALSE; } From 3d1b47803f2c0efde9fea20ada0480d84900d5a6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 25 Apr 2017 11:55:01 -0400 Subject: [PATCH 22/94] repo: More porting to new style I was planning to change some of the object loading code in the future, so here's some porting. Note that I rewrote `_ostree_repo_has_loose_object()` since it used an error return across multiple functions. Honestly I'm not sure about this `TEMP_FAILURE_RETRY()` business... in reality we're going to end up with a ton of code linked in process that doesn't do it. Unix sucks =( But I'm keeping what was there out of consistency. Closes: #809 Approved by: jlebon --- src/libostree/ostree-repo.c | 286 ++++++++++++------------------------ 1 file changed, 98 insertions(+), 188 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2788d330..5a2d94f2 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2606,27 +2606,19 @@ query_info_for_bare_content_object (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; struct stat stbuf; - int res; - g_autoptr(GFileInfo) ret_info = NULL; - do - res = fstatat (self->objects_dir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) + if (TEMP_FAILURE_RETRY (fstatat (self->objects_dir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) { if (errno == ENOENT) { *out_info = NULL; - ret = TRUE; - goto out; + return TRUE; } - glnx_set_error_from_errno (error); - goto out; + return glnx_throw_errno (error); } - ret_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); + g_autoptr(GFileInfo) ret_info = _ostree_header_gfile_info_new (stbuf.st_mode, stbuf.st_uid, stbuf.st_gid); if (S_ISREG (stbuf.st_mode)) { @@ -2636,20 +2628,13 @@ query_info_for_bare_content_object (OstreeRepo *self, { if (!ot_readlinkat_gfile_info (self->objects_dir_fd, loose_path_buf, ret_info, cancellable, error)) - goto out; + return FALSE; } else - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not a regular file or symlink: %s", loose_path_buf); - goto out; - } + return glnx_throw_errno_prefix (error, "Not a regular file or symlink: %s", loose_path_buf); - ret = TRUE; - if (out_info) - *out_info = g_steal_pointer (&ret_info); - out: - return ret; + ot_transfer_out_value (out_info, &ret_info); + return TRUE; } static GVariant * @@ -2727,16 +2712,14 @@ ostree_repo_load_file (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean found = FALSE; - OstreeRepoMode repo_mode; g_autoptr(GInputStream) ret_input = NULL; g_autoptr(GFileInfo) ret_file_info = NULL; g_autoptr(GVariant) ret_xattrs = NULL; + + OstreeRepoMode repo_mode = ostree_repo_get_mode (self); + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - - repo_mode = ostree_repo_get_mode (self); - _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, repo_mode); if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2) @@ -2747,29 +2730,29 @@ ostree_repo_load_file (OstreeRepo *self, if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, &fd, error)) - goto out; + return FALSE; if (fd < 0 && self->commit_stagedir_fd != -1) { if (!ot_openat_ignore_enoent (self->commit_stagedir_fd, loose_path_buf, &fd, error)) - goto out; + return FALSE; } if (fd != -1) { tmp_stream = g_unix_input_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ - + if (!glnx_stream_fstat ((GFileDescriptorBased*) tmp_stream, &stbuf, error)) - goto out; - + return FALSE; + if (!ostree_content_stream_parse (TRUE, tmp_stream, stbuf.st_size, TRUE, out_input ? &ret_input : NULL, &ret_file_info, &ret_xattrs, cancellable, error)) - goto out; + return FALSE; found = TRUE; } @@ -2779,7 +2762,7 @@ ostree_repo_load_file (OstreeRepo *self, if (!query_info_for_bare_content_object (self, loose_path_buf, &ret_file_info, cancellable, error)) - goto out; + return FALSE; if (ret_file_info) { @@ -2798,14 +2781,11 @@ ostree_repo_load_file (OstreeRepo *self, */ fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); bytes = glnx_fgetxattr_bytes (fd, "user.ostreemeta", error); if (bytes == NULL) - goto out; + return FALSE; metadata = g_variant_new_from_bytes (OSTREE_FILEMETA_GVARIANT_FORMAT, bytes, FALSE); @@ -2835,7 +2815,7 @@ ostree_repo_load_file (OstreeRepo *self, if (!g_input_stream_read_all (target_input, targetbuf, sizeof (targetbuf), &target_size, cancellable, error)) - goto out; + return FALSE; g_file_info_set_symlink_target (ret_file_info, targetbuf); } @@ -2855,10 +2835,7 @@ ostree_repo_load_file (OstreeRepo *self, { fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); ret_input = g_unix_input_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ @@ -2882,10 +2859,7 @@ ostree_repo_load_file (OstreeRepo *self, fd = openat (self->objects_dir_fd, loose_path_buf, O_RDONLY | O_CLOEXEC); if (fd < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); if (out_xattrs) { @@ -2893,7 +2867,7 @@ ostree_repo_load_file (OstreeRepo *self, ret_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0)); else if (!glnx_fd_get_all_xattrs (fd, &ret_xattrs, cancellable, error)) - goto out; + return FALSE; } if (out_input) @@ -2910,7 +2884,7 @@ ostree_repo_load_file (OstreeRepo *self, else if (!glnx_dfd_name_get_all_xattrs (self->objects_dir_fd, loose_path_buf, &ret_xattrs, cancellable, error)) - goto out; + return FALSE; } } } @@ -2925,22 +2899,20 @@ ostree_repo_load_file (OstreeRepo *self, out_file_info ? &ret_file_info : NULL, out_xattrs ? &ret_xattrs : NULL, cancellable, error)) - goto out; + return FALSE; } else { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, "Couldn't find file object '%s'", checksum); - goto out; + return FALSE; } } - ret = TRUE; ot_transfer_out_value (out_input, &ret_input); ot_transfer_out_value (out_file_info, &ret_file_info); ot_transfer_out_value (out_xattrs, &ret_xattrs); - out: - return ret; + return TRUE; } /** @@ -2965,7 +2937,6 @@ ostree_repo_load_object_stream (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; guint64 size; g_autoptr(GInputStream) ret_input = NULL; @@ -2974,7 +2945,7 @@ ostree_repo_load_object_stream (OstreeRepo *self, if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL, &ret_input, &size, cancellable, error)) - goto out; + return FALSE; } else { @@ -2984,19 +2955,17 @@ ostree_repo_load_object_stream (OstreeRepo *self, if (!ostree_repo_load_file (self, checksum, &input, &finfo, &xattrs, cancellable, error)) - goto out; + return FALSE; if (!ostree_raw_file_to_content_stream (input, finfo, xattrs, &ret_input, &size, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; ot_transfer_out_value (out_input, &ret_input); *out_size = size; - out: - return ret; + return TRUE; } /* @@ -3014,41 +2983,34 @@ _ostree_repo_has_loose_object (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - struct stat stbuf; - int res = -1; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (loose_path_buf, checksum, objtype, self->mode); - if (self->commit_stagedir_fd != -1) + gboolean found = FALSE; + /* It's easier to share code if we make this an array */ + const int dfd_searches[] = { self->commit_stagedir_fd, self->objects_dir_fd }; + for (guint i = 0; i < G_N_ELEMENTS (dfd_searches); i++) { - do - res = fstatat (self->commit_stagedir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1 && errno != ENOENT) + int dfd = dfd_searches[i]; + if (dfd == -1) + continue; + struct stat stbuf; + if (TEMP_FAILURE_RETRY (fstatat (dfd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) { - glnx_set_error_from_errno (error); - goto out; + if (errno == ENOENT) + ; /* Next dfd */ + else + return glnx_throw_errno (error); + } + else + { + found = TRUE; + break; } } - if (res < 0) - { - do - res = fstatat (self->objects_dir_fd, loose_path_buf, &stbuf, AT_SYMLINK_NOFOLLOW); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1 && errno != ENOENT) - { - glnx_set_error_from_errno (error); - goto out; - } - } - - ret = TRUE; - *out_is_stored = (res != -1); -out: - return ret; + *out_is_stored = found; + return TRUE; } /** @@ -3073,12 +3035,11 @@ ostree_repo_has_object (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean ret_have_object; if (!_ostree_repo_has_loose_object (self, checksum, objtype, &ret_have_object, cancellable, error)) - goto out; + return FALSE; /* In the future, here is where we would also look up in metadata pack files */ @@ -3086,14 +3047,12 @@ ostree_repo_has_object (OstreeRepo *self, { if (!ostree_repo_has_object (self->parent_repo, objtype, checksum, &ret_have_object, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; if (out_have_object) *out_have_object = ret_have_object; - out: - return ret; + return TRUE; } /** @@ -3115,10 +3074,7 @@ ostree_repo_delete_object (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - int res; char loose_path[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (loose_path, sha256, objtype, self->mode); if (objtype == OSTREE_OBJECT_TYPE_COMMIT) @@ -3127,27 +3083,15 @@ ostree_repo_delete_object (OstreeRepo *self, _ostree_loose_path (meta_loose, sha256, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode); - do - res = unlinkat (self->objects_dir_fd, meta_loose, 0); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (res == -1) + if (TEMP_FAILURE_RETRY (unlinkat (self->objects_dir_fd, meta_loose, 0)) < 0) { if (G_UNLIKELY (errno != ENOENT)) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "unlinkat(%s)", meta_loose); } } - do - res = unlinkat (self->objects_dir_fd, loose_path, 0); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) - { - glnx_set_prefix_error_from_errno (error, "Deleting object %s.%s", sha256, ostree_object_type_to_string (objtype)); - goto out; - } + if (TEMP_FAILURE_RETRY (unlinkat (self->objects_dir_fd, loose_path, 0)) < 0) + return glnx_throw_errno_prefix (error, "Deleting object %s.%s", sha256, ostree_object_type_to_string (objtype)); /* If the repository is configured to use tombstone commits, create one when deleting a commit. */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) @@ -3156,7 +3100,7 @@ ostree_repo_delete_object (OstreeRepo *self, GKeyFile *readonly_config = ostree_repo_get_config (self); if (!ot_keyfile_get_boolean_with_default (readonly_config, "core", "tombstone-commits", FALSE, &tombstone_commits, error)) - goto out; + return FALSE; if (tombstone_commits) { @@ -3172,13 +3116,11 @@ ostree_repo_delete_object (OstreeRepo *self, variant, cancellable, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -3188,25 +3130,21 @@ copy_detached_metadata (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) detached_meta = NULL; - if (!ostree_repo_read_commit_detached_metadata (source, checksum, &detached_meta, cancellable, error)) - goto out; + return FALSE; if (detached_meta) { if (!ostree_repo_write_commit_detached_metadata (self, checksum, detached_meta, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -3218,14 +3156,13 @@ import_one_object_copy (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; guint64 length; g_autoptr(GInputStream) object_stream = NULL; if (!ostree_repo_load_object_stream (source, objtype, checksum, &object_stream, &length, cancellable, error)) - goto out; + return FALSE; if (objtype == OSTREE_OBJECT_TYPE_FILE) { @@ -3234,7 +3171,7 @@ import_one_object_copy (OstreeRepo *self, if (!ostree_repo_write_content_trusted (self, checksum, object_stream, length, cancellable, error)) - goto out; + return FALSE; } else { @@ -3243,7 +3180,7 @@ import_one_object_copy (OstreeRepo *self, object_stream, length, &real_csum, cancellable, error)) - goto out; + return FALSE; } } else @@ -3251,7 +3188,7 @@ import_one_object_copy (OstreeRepo *self, if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!copy_detached_metadata (self, source, checksum, cancellable, error)) - goto out; + return FALSE; } if (trusted) @@ -3259,7 +3196,7 @@ import_one_object_copy (OstreeRepo *self, if (!ostree_repo_write_metadata_stream_trusted (self, objtype, checksum, object_stream, length, cancellable, error)) - goto out; + return FALSE; } else { @@ -3268,19 +3205,17 @@ import_one_object_copy (OstreeRepo *self, if (!ostree_repo_load_variant (source, objtype, checksum, &variant, error)) - goto out; + return FALSE; if (!ostree_repo_write_metadata (self, objtype, checksum, variant, &real_csum, cancellable, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -3292,44 +3227,36 @@ import_one_object_link (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; - _ostree_loose_path (loose_path_buf, checksum, objtype, self->mode); if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path_buf, cancellable, error)) - goto out; + return FALSE; *out_was_supported = TRUE; if (linkat (source->objects_dir_fd, loose_path_buf, self->objects_dir_fd, loose_path_buf, 0) != 0) { if (errno == EEXIST) - { - ret = TRUE; - } + return TRUE; else if (errno == EMLINK || errno == EXDEV || errno == EPERM) { /* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do the * optimization of hardlinking instead of copying. */ *out_was_supported = FALSE; - ret = TRUE; + return TRUE; } else - glnx_set_error_from_errno (error); - - goto out; + return glnx_throw_errno (error); } if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!copy_detached_metadata (self, source, checksum, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -3352,7 +3279,7 @@ gboolean ostree_repo_import_object_from (OstreeRepo *self, OstreeRepo *source, OstreeObjectType objtype, - const char *checksum, + const char *checksum, GCancellable *cancellable, GError **error) { @@ -3387,7 +3314,6 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; gboolean hardlink_was_supported = FALSE; if (trusted && /* Don't hardlink into untrusted remotes */ @@ -3396,7 +3322,7 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, if (!import_one_object_link (self, source, checksum, objtype, &hardlink_was_supported, cancellable, error)) - goto out; + return FALSE; } if (!hardlink_was_supported) @@ -3405,19 +3331,17 @@ ostree_repo_import_object_from_with_trust (OstreeRepo *self, if (!ostree_repo_has_object (self, objtype, checksum, &has_object, cancellable, error)) - goto out; + return FALSE; if (!has_object) { if (!import_one_object_copy (self, source, checksum, objtype, trusted, cancellable, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } @@ -3442,15 +3366,10 @@ ostree_repo_query_object_storage_size (OstreeRepo *self, GError **error) { char loose_path[_OSTREE_LOOSE_PATH_MAX]; - int res; - struct stat stbuf; - _ostree_loose_path (loose_path, sha256, objtype, self->mode); - do - res = fstatat (self->objects_dir_fd, loose_path, &stbuf, AT_SYMLINK_NOFOLLOW); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + struct stat stbuf; + if (TEMP_FAILURE_RETRY (fstatat (self->objects_dir_fd, loose_path, &stbuf, AT_SYMLINK_NOFOLLOW)) < 0) return glnx_throw_errno_prefix (error, "Querying object %s.%s", sha256, ostree_object_type_to_string (objtype)); *out_size = stbuf.st_size; @@ -3624,31 +3543,26 @@ ostree_repo_list_commit_objects_starting_with (OstreeRepo *self GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GHashTable) ret_commits = NULL; - g_return_val_if_fail (error == NULL || *error == NULL, FALSE); g_return_val_if_fail (self->inited, FALSE); - ret_commits = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, - (GDestroyNotify) g_variant_unref, - (GDestroyNotify) g_variant_unref); + g_autoptr(GHashTable) ret_commits = + g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, + (GDestroyNotify) g_variant_unref, + (GDestroyNotify) g_variant_unref); if (!list_loose_objects (self, ret_commits, start, cancellable, error)) - goto out; - + return FALSE; if (self->parent_repo) { if (!list_loose_objects (self->parent_repo, ret_commits, start, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; ot_transfer_out_value (out_commits, &ret_commits); - out: - return ret; + return TRUE; } /** @@ -3953,29 +3867,25 @@ ostree_repo_append_gpg_signature (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) metadata = NULL; - g_autoptr(GVariant) new_metadata = NULL; - if (!ostree_repo_read_commit_detached_metadata (self, commit_checksum, &metadata, cancellable, error)) - goto out; + return FALSE; - new_metadata = _ostree_detached_metadata_append_gpg_sig (metadata, signature_bytes); + g_autoptr(GVariant) new_metadata = + _ostree_detached_metadata_append_gpg_sig (metadata, signature_bytes); if (!ostree_repo_write_commit_detached_metadata (self, commit_checksum, new_metadata, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean From 1ceb17ff118a00d54bd2eaaf6f87032fa1179bbe Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 26 Apr 2017 13:03:42 +0200 Subject: [PATCH 23/94] apidoc: Add missing enums to sections file So they and their values show up in the documentation. Closes: #812 Approved by: cgwalters --- apidoc/ostree-sections.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 07dbaf1f..e6ed4c8a 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -316,11 +316,14 @@ ostree_repo_write_content_trusted ostree_repo_write_content_async ostree_repo_write_content_finish ostree_repo_resolve_rev +OstreeRepoResolveRevExtFlags ostree_repo_resolve_rev_ext ostree_repo_list_refs +OstreeRepoListRefsExtFlags ostree_repo_list_refs_ext ostree_repo_remote_list_refs ostree_repo_load_variant +OstreeRepoCommitState ostree_repo_load_commit ostree_repo_load_variant_if_exists ostree_repo_load_file @@ -376,8 +379,10 @@ ostree_repo_commit_traverse_iter_cleanup ostree_repo_commit_traverse_iter_clear ostree_repo_commit_traverse_iter_get_dir ostree_repo_commit_traverse_iter_get_file +OstreeRepoCommitTraverseFlags ostree_repo_commit_traverse_iter_init_commit ostree_repo_commit_traverse_iter_init_dirtree +OstreeRepoCommitIterResult ostree_repo_commit_traverse_iter_next OstreeRepoPruneFlags ostree_repo_prune From 20b0836ec856675416b615315b94d80bad871af8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 25 Apr 2017 17:27:32 -0400 Subject: [PATCH 24/94] repo: Fix incorrect use of errno() error throwing I happened to glance at the top of my most recent patch and noticed that I used an `throw_errno()` function in a non-errno place. I scanned the patch for other instances of this but didn't find one. Closes: #811 Approved by: jlebon --- src/libostree/ostree-repo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5a2d94f2..ec2d8c83 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2631,7 +2631,7 @@ query_info_for_bare_content_object (OstreeRepo *self, return FALSE; } else - return glnx_throw_errno_prefix (error, "Not a regular file or symlink: %s", loose_path_buf); + return glnx_throw (error, "Not a regular file or symlink: %s", loose_path_buf); ot_transfer_out_value (out_info, &ret_info); return TRUE; From ceb8851806c07a3349d99633fbb6d7d8769e6f7f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 26 Apr 2017 16:22:32 -0400 Subject: [PATCH 25/94] lib/sepolicy: Convert to new code style I have a fix I want to make here and it's really hard to write the old style code now. Closes: #815 Approved by: jlebon --- src/libostree/ostree-sepolicy.c | 155 ++++++++++---------------------- 1 file changed, 46 insertions(+), 109 deletions(-) diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index 84b2d5c7..45a40043 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -198,24 +198,21 @@ get_policy_checksum (char **out_csum, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const char *binary_policy_path = selinux_binary_policy_path (); const char *binfile_prefix = glnx_basename (binary_policy_path); g_autofree char *bindir_path = g_path_get_dirname (binary_policy_path); - glnx_fd_close int bindir_dfd = -1; g_autofree char *best_policy = NULL; int best_version = 0; - g_auto(GLnxDirFdIterator) dfd_iter = { 0,}; - + glnx_fd_close int bindir_dfd = -1; if (!glnx_opendirat (AT_FDCWD, bindir_path, TRUE, &bindir_dfd, error)) - goto out; + return FALSE; + g_auto(GLnxDirFdIterator) dfd_iter = { 0,}; if (!glnx_dirfd_iterator_init_at (bindir_dfd, ".", FALSE, &dfd_iter, error)) - goto out; + return FALSE; while (TRUE) { @@ -223,8 +220,7 @@ get_policy_checksum (char **out_csum, if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) - goto out; - + return FALSE; if (dent == NULL) break; @@ -259,20 +255,14 @@ get_policy_checksum (char **out_csum, } if (!best_policy) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Could not find binary policy file"); - goto out; - } + return glnx_throw (error, "Could not find binary policy file"); *out_csum = ot_checksum_file_at (bindir_dfd, best_policy, G_CHECKSUM_SHA256, cancellable, error); if (*out_csum == NULL) - goto out; + return FALSE; - ret = TRUE; -out: - return ret; + return TRUE; } #endif @@ -283,20 +273,14 @@ initable_init (GInitable *initable, GError **error) { #ifdef HAVE_SELINUX - gboolean ret = FALSE; OstreeSePolicy *self = OSTREE_SEPOLICY (initable); - g_autoptr(GFile) path = NULL; - g_autoptr(GFile) etc_selinux_dir = NULL; - g_autoptr(GFile) policy_config_path = NULL; - g_autoptr(GFile) policy_root = NULL; - g_autoptr(GFileInputStream) filein = NULL; - g_autoptr(GDataInputStream) datain = NULL; gboolean enabled = FALSE; g_autofree char *policytype = NULL; const char *selinux_prefix = "SELINUX="; const char *selinuxtype_prefix = "SELINUXTYPE="; /* TODO - use this below */ + g_autoptr(GFile) path = NULL; if (self->rootfs_dfd != -1) path = ot_fdrel_to_gfile (self->rootfs_dfd, "."); else if (self->path) @@ -306,45 +290,44 @@ initable_init (GInitable *initable, /* TODO - use this below */ if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (self->path), TRUE, &self->rootfs_dfd_owned, error)) - goto out; + return FALSE; self->rootfs_dfd = self->rootfs_dfd_owned; #endif } else g_assert_not_reached (); - etc_selinux_dir = g_file_resolve_relative_path (path, "etc/selinux"); + g_autoptr(GFile) etc_selinux_dir = g_file_resolve_relative_path (path, "etc/selinux"); if (!g_file_query_exists (etc_selinux_dir, NULL)) { g_object_unref (etc_selinux_dir); etc_selinux_dir = g_file_resolve_relative_path (path, "usr/etc/selinux"); } - policy_config_path = g_file_get_child (etc_selinux_dir, "config"); + g_autoptr(GFile) policy_config_path = g_file_get_child (etc_selinux_dir, "config"); + g_autoptr(GFile) policy_root = NULL; if (g_file_query_exists (policy_config_path, NULL)) { - filein = g_file_read (policy_config_path, cancellable, error); - if (!filein) - goto out; + g_autoptr(GFileInputStream) filein = filein = g_file_read (policy_config_path, cancellable, error); - datain = g_data_input_stream_new ((GInputStream*)filein); + if (!filein) + return FALSE; + + g_autoptr(GDataInputStream) datain = g_data_input_stream_new ((GInputStream*)filein); while (TRUE) { gsize len; - GError *temp_error = NULL; + g_autoptr(GError) temp_error = NULL; g_autofree char *line = g_data_input_stream_read_line_utf8 (datain, &len, cancellable, &temp_error); - + if (temp_error) - { - g_propagate_error (error, temp_error); - goto out; - } + return g_propagate_error (error, g_steal_pointer (&temp_error)), FALSE; if (!line) break; - + if (g_str_has_prefix (line, selinuxtype_prefix)) { policytype = g_strstrip (g_strdup (line + strlen (selinuxtype_prefix))); @@ -363,56 +346,32 @@ initable_init (GInitable *initable, if (enabled) { self->runtime_enabled = is_selinux_enabled () == 1; + const char *policy_rootpath = gs_file_get_path_cached (policy_root); g_setenv ("LIBSELINUX_DISABLE_PCRE_PRECOMPILED", "1", FALSE); - if (selinux_set_policy_root (gs_file_get_path_cached (policy_root)) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "selinux_set_policy_root(%s): %s", - gs_file_get_path_cached (etc_selinux_dir), - strerror (errno)); - goto out; - } + if (selinux_set_policy_root (policy_rootpath) != 0) + return glnx_throw_errno_prefix (error, "selinux_set_policy_root(%s)", policy_rootpath); self->selinux_hnd = selabel_open (SELABEL_CTX_FILE, NULL, 0); if (!self->selinux_hnd) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "With policy root '%s': selabel_open(SELABEL_CTX_FILE): %s", - gs_file_get_path_cached (etc_selinux_dir), - strerror (errno)); - goto out; - } + return glnx_throw_errno_prefix (error, "With policy root '%s': selabel_open(SELABEL_CTX_FILE)", + policy_rootpath); - { - char *con = NULL; - if (selabel_lookup_raw (self->selinux_hnd, &con, "/", 0755) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "With policy root '%s': Failed to look up context of /: %s", - gs_file_get_path_cached (etc_selinux_dir), - strerror (errno)); - goto out; - } - freecon (con); - } + char *con = NULL; + if (selabel_lookup_raw (self->selinux_hnd, &con, "/", 0755) != 0) + return glnx_throw_errno_prefix (error, "With policy root '%s': Failed to look up context of /", + policy_rootpath); + freecon (con); if (!get_policy_checksum (&self->selinux_policy_csum, cancellable, error)) - { - g_prefix_error (error, "While calculating SELinux checksum: "); - goto out; - } + return g_prefix_error (error, "While calculating SELinux checksum: "), FALSE; self->selinux_policy_name = g_steal_pointer (&policytype); self->selinux_policy_root = g_object_ref (etc_selinux_dir); } - ret = TRUE; - out: - return ret; -#else - return TRUE; #endif + return TRUE; } static void @@ -580,11 +539,7 @@ ostree_sepolicy_restorecon (OstreeSePolicy *self, GError **error) { #ifdef HAVE_SELINUX - gboolean ret = FALSE; g_autoptr(GFileInfo) src_info = NULL; - g_autofree char *label = NULL; - gboolean do_relabel = TRUE; - if (info != NULL) src_info = g_object_ref (info); else @@ -593,9 +548,10 @@ ostree_sepolicy_restorecon (OstreeSePolicy *self, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error); if (!src_info) - goto out; + return FALSE; } + gboolean do_relabel = TRUE; if (flags & OSTREE_SEPOLICY_RESTORECON_FLAGS_KEEP_EXISTING) { char *existing_con = NULL; @@ -607,42 +563,31 @@ ostree_sepolicy_restorecon (OstreeSePolicy *self, } } + g_autofree char *label = NULL; if (do_relabel) { - if (!ostree_sepolicy_get_label (self, path, + if (!ostree_sepolicy_get_label (self, path, g_file_info_get_attribute_uint32 (src_info, "unix::mode"), &label, cancellable, error)) - goto out; + return FALSE; if (!label) { if (!(flags & OSTREE_SEPOLICY_RESTORECON_FLAGS_ALLOW_NOLABEL)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No label found for '%s'", path); - goto out; - } + return glnx_throw (error, "No label found for '%s'", path); } else { - int res = lsetfilecon (gs_file_get_path_cached (target), label); - if (res != 0) - { - glnx_set_error_from_errno (error); - goto out; - } + if (lsetfilecon (gs_file_get_path_cached (target), label) < 0) + return glnx_throw_errno_prefix (error, "lsetfilecon"); } } - ret = TRUE; if (out_new_label) *out_new_label = g_steal_pointer (&label); - out: - return ret; -#else - return TRUE; #endif + return TRUE; } /** @@ -660,7 +605,6 @@ ostree_sepolicy_setfscreatecon (OstreeSePolicy *self, GError **error) { #ifdef HAVE_SELINUX - gboolean ret = FALSE; g_autofree char *label = NULL; /* setfscreatecon() will bomb out if the host has SELinux disabled, @@ -673,20 +617,13 @@ ostree_sepolicy_setfscreatecon (OstreeSePolicy *self, return TRUE; if (!ostree_sepolicy_get_label (self, path, mode, &label, NULL, error)) - goto out; + return FALSE; if (setfscreatecon_raw (label) != 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "setfscreatecon"); - ret = TRUE; - out: - return ret; -#else - return TRUE; #endif + return TRUE; } /** From 90b24a8d43a978da57add31b12f2b71c5179cfdc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 26 Apr 2017 16:54:52 -0400 Subject: [PATCH 26/94] sepolicy: Cache the value of is_selinux_enabled() to work around bug This fixes a regression from: https://github.com/ostreedev/ostree/pull/797 which is really due to an underlying bug in libselinux which we're working around: http://marc.info/?l=selinux&m=149323809332417&w=2 We drop the per-policy instance variable, since the SELinux state is *really* per-kernel. Closes: https://github.com/ostreedev/ostree/issues/814 Closes: #815 Approved by: jlebon --- src/libostree/ostree-sepolicy.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index 45a40043..dd0567aa 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -47,8 +47,6 @@ struct OstreeSePolicy { int rootfs_dfd_owned; GFile *path; - gboolean runtime_enabled; - #ifdef HAVE_SELINUX GFile *selinux_policy_root; struct selabel_handle *selinux_hnd; @@ -267,6 +265,23 @@ get_policy_checksum (char **out_csum, #endif + +/* Workaround for http://marc.info/?l=selinux&m=149323809332417&w=2 */ +#ifdef HAVE_SELINUX +static gboolean +cached_is_selinux_enabled (void) +{ + static gsize initialized; + static gboolean cached_enabled; + if (g_once_init_enter (&initialized)) + { + cached_enabled = is_selinux_enabled () == 1; + g_once_init_leave (&initialized, 1); + } + return cached_enabled; +} +#endif + static gboolean initable_init (GInitable *initable, GCancellable *cancellable, @@ -279,6 +294,11 @@ initable_init (GInitable *initable, const char *selinux_prefix = "SELINUX="; const char *selinuxtype_prefix = "SELINUXTYPE="; + /* First thing here, call is_selinux_enabled() to prime the cache. See the + * link above for more information why. + */ + (void) cached_is_selinux_enabled (); + /* TODO - use this below */ g_autoptr(GFile) path = NULL; if (self->rootfs_dfd != -1) @@ -345,7 +365,6 @@ initable_init (GInitable *initable, if (enabled) { - self->runtime_enabled = is_selinux_enabled () == 1; const char *policy_rootpath = gs_file_get_path_cached (policy_root); g_setenv ("LIBSELINUX_DISABLE_PCRE_PRECOMPILED", "1", FALSE); @@ -613,7 +632,7 @@ ostree_sepolicy_setfscreatecon (OstreeSePolicy *self, * request. To correctly handle the case of disabled host but * enabled target will require nontrivial work. */ - if (!self->runtime_enabled) + if (!cached_is_selinux_enabled ()) return TRUE; if (!ostree_sepolicy_get_label (self, path, mode, &label, NULL, error)) From c41860059490c9b29846fd492e938aec2f5dc976 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 25 Apr 2017 19:14:12 +0100 Subject: [PATCH 27/94] ostree: Use G_OPTION_ARG_FILENAME where appropriate Instead of using G_OPTION_ARG_STRING, use G_OPTION_ARG_FILENAME, which handles filename encoding conversion differently from the locale conversion which G_OPTION_ARG_STRING. This will fix argument handling on systems where the filename encoding is not the same as the locale encoding (which is fairly unlikely since most systems use UTF-8). Signed-off-by: Philip Withnall Closes: #810 Approved by: cgwalters --- src/ostree/ot-builtin-checkout.c | 2 +- src/ostree/ot-builtin-commit.c | 4 ++-- src/ostree/ot-builtin-export.c | 6 +++--- src/ostree/ot-builtin-gpg-sign.c | 2 +- src/ostree/ot-builtin-pull.c | 4 ++-- src/ostree/ot-builtin-show.c | 2 +- src/ostree/ot-builtin-static-delta.c | 2 +- src/ostree/ot-builtin-summary.c | 2 +- src/ostree/ot-main.c | 4 ++-- src/ostree/ot-remote-builtin-refs.c | 2 +- src/ostree/ot-remote-builtin-summary.c | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 9ba804f8..8ffe5bb7 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -63,7 +63,7 @@ parse_fsync_cb (const char *option_name, static GOptionEntry options[] = { { "user-mode", 'U', 0, G_OPTION_ARG_NONE, &opt_user_mode, "Do not change file ownership or initialize extended attributes", NULL }, { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL }, - { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, + { "subpath", 0, 0, G_OPTION_ARG_FILENAME, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL }, { "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL }, { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL }, diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index e6738c89..53dab6d4 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -76,7 +76,7 @@ static GOptionEntry options[] = { { "parent", 0, 0, G_OPTION_ARG_STRING, &opt_parent, "Parent ref, or \"none\"", "REF" }, { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject", "SUBJECT" }, { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "BODY" }, - { "body-file", 'F', 0, G_OPTION_ARG_STRING, &opt_body_file, "Commit message from FILE path", "FILE" }, + { "body-file", 'F', 0, G_OPTION_ARG_FILENAME, &opt_body_file, "Commit message from FILE path", "FILE" }, { "editor", 'e', 0, G_OPTION_ARG_NONE, &opt_editor, "Use an editor to write the commit message", NULL }, { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" }, { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, @@ -94,7 +94,7 @@ static GOptionEntry options[] = { { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" }, { "table-output", 0, 0, G_OPTION_ARG_NONE, &opt_table_output, "Output more information in a KEY: VALUE format", NULL }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"}, - { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, + { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, { "generate-sizes", 0, 0, G_OPTION_ARG_NONE, &opt_generate_sizes, "Generate size information along with commit metadata", NULL }, { "disable-fsync", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "fsync", 0, 0, G_OPTION_ARG_CALLBACK, parse_fsync_cb, "Specify how to invoke fsync()", "POLICY" }, diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c index f3cdfbe0..2d40112a 100644 --- a/src/ostree/ot-builtin-export.c +++ b/src/ostree/ot-builtin-export.c @@ -39,9 +39,9 @@ static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, - { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, - { "prefix", 0, 0, G_OPTION_ARG_STRING, &opt_prefix, "Add PATH as prefix to archive pathnames", "PATH" }, - { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, + { "subpath", 0, 0, G_OPTION_ARG_FILENAME, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, + { "prefix", 0, 0, G_OPTION_ARG_FILENAME, &opt_prefix, "Add PATH as prefix to archive pathnames", "PATH" }, + { "output", 'o', 0, G_OPTION_ARG_FILENAME, &opt_output_path, "Output to PATH ", "PATH" }, { NULL } }; diff --git a/src/ostree/ot-builtin-gpg-sign.c b/src/ostree/ot-builtin-gpg-sign.c index c19ec682..0ef2be8b 100644 --- a/src/ostree/ot-builtin-gpg-sign.c +++ b/src/ostree/ot-builtin-gpg-sign.c @@ -32,7 +32,7 @@ static char *opt_gpg_homedir; static GOptionEntry options[] = { { "delete", 'd', 0, G_OPTION_ARG_NONE, &opt_delete, "Delete signatures having any of the GPG KEY-IDs" }, - { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR" }, + { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR" }, { NULL } }; diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 4287afcd..a7826542 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -43,12 +43,12 @@ static char* opt_url; static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, - { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, + { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, - { "subpath", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_subpaths, "Only pull the provided subpath(s)", NULL }, + { "subpath", 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_subpaths, "Only pull the provided subpath(s)", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust (local) sources", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, diff --git a/src/ostree/ot-builtin-show.c b/src/ostree/ot-builtin-show.c index 3aaa2303..dcd9090c 100644 --- a/src/ostree/ot-builtin-show.c +++ b/src/ostree/ot-builtin-show.c @@ -42,7 +42,7 @@ static GOptionEntry options[] = { { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key", "KEY" }, { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key", "KEY" }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" }, - { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, + { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, { "gpg-verify-remote", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_verify_remote, "Use REMOTE name for GPG configuration", "REMOTE"}, { NULL } }; diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 1019f06f..c98436af 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -72,7 +72,7 @@ static GOptionEntry generate_options[] = { { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL}, { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, - { "filename", 0, 0, G_OPTION_ARG_STRING, &opt_filename, "Write the delta content to PATH (a directory). If not specified, the OSTree repository is used", "PATH"}, + { "filename", 0, 0, G_OPTION_ARG_FILENAME, &opt_filename, "Write the delta content to PATH (a directory). If not specified, the OSTree repository is used", "PATH"}, { NULL } }; diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index 04ba84e8..fa037400 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -32,7 +32,7 @@ static char *opt_gpg_homedir; static GOptionEntry options[] = { { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, - { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, + { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, { NULL } }; diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index c00b4918..efc240a7 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -45,14 +45,14 @@ static GOptionEntry global_entries[] = { }; static GOptionEntry repo_entry[] = { - { "repo", 0, 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository (defaults to /sysroot/ostree/repo)", "PATH" }, + { "repo", 0, 0, G_OPTION_ARG_FILENAME, &opt_repo, "Path to OSTree repository (defaults to /sysroot/ostree/repo)", "PATH" }, { NULL } }; static GOptionEntry global_admin_entries[] = { /* No description since it's hidden from --help output. */ { "print-current-dir", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_print_current_dir, NULL, NULL }, - { "sysroot", 0, 0, G_OPTION_ARG_STRING, &opt_sysroot, "Create a new OSTree sysroot at PATH", "PATH" }, + { "sysroot", 0, 0, G_OPTION_ARG_FILENAME, &opt_sysroot, "Create a new OSTree sysroot at PATH", "PATH" }, { NULL } }; diff --git a/src/ostree/ot-remote-builtin-refs.c b/src/ostree/ot-remote-builtin-refs.c index f5b3afe3..9e6ee144 100644 --- a/src/ostree/ot-remote-builtin-refs.c +++ b/src/ostree/ot-remote-builtin-refs.c @@ -28,7 +28,7 @@ static char* opt_cache_dir; static GOptionEntry option_entries[] = { - { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, + { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, { NULL } }; diff --git a/src/ostree/ot-remote-builtin-summary.c b/src/ostree/ot-remote-builtin-summary.c index b4d99254..5ddcf69b 100644 --- a/src/ostree/ot-remote-builtin-summary.c +++ b/src/ostree/ot-remote-builtin-summary.c @@ -31,7 +31,7 @@ static gboolean opt_raw; static char* opt_cache_dir; static GOptionEntry option_entries[] = { - { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, + { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data", NULL }, { NULL } }; From d37acd3007be19aa8c4e3b525e0d1cf8a281322d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 19 Apr 2017 00:02:51 +0100 Subject: [PATCH 28/94] tests: Ignore some standard automake check output files Signed-off-by: Philip Withnall Closes: #810 Approved by: cgwalters --- tests/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/.gitignore b/tests/.gitignore index afa0b233..6fc06881 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,4 +1,7 @@ !Makefile +*.log +*.test +*.trs ostree-http-server run-apache tmpdir-lifecycle From 838cbab58593e1c94b158714e26cd6433717f32e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 26 Apr 2017 17:38:14 -0400 Subject: [PATCH 29/94] lib/checkout: Use TEMP_FAILURE_RETRY() I'm still not sure it's worth using, but it's easier on the eyes for sure. Closes: #816 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 41 ++++++---------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 2fd0b46a..fa29699b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -68,11 +68,7 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, if (!self->disable_fsync) { - int res; - do - res = fsync (fd); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (fsync (fd)) < 0) return glnx_throw_errno (error); } @@ -126,19 +122,11 @@ write_regular_file_content (OstreeRepo *self, if (mode != OSTREE_REPO_CHECKOUT_MODE_USER) { - int res; - do - res = fchown (fd, - g_file_info_get_attribute_uint32 (file_info, "unix::uid"), - g_file_info_get_attribute_uint32 (file_info, "unix::gid")); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (fchown (fd, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), + g_file_info_get_attribute_uint32 (file_info, "unix::gid"))) < 0) return glnx_throw_errno (error); - do - res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"))) < 0) return glnx_throw_errno (error); if (xattrs) @@ -598,7 +586,6 @@ checkout_tree_at_recurse (OstreeRepo *self, GError **error) { gboolean did_exist = FALSE; - int res; const gboolean sepolicy_enabled = options->sepolicy && !self->disable_xattrs; g_autoptr(GVariant) xattrs = NULL; g_autoptr(GVariant) modified_xattrs = NULL; @@ -733,22 +720,15 @@ checkout_tree_at_recurse (OstreeRepo *self, */ if (!did_exist) { - do - res = fchmod (destination_dfd, - g_file_info_get_attribute_uint32 (source_info, "unix::mode")); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (fchmod (destination_dfd, g_file_info_get_attribute_uint32 (source_info, "unix::mode"))) < 0) return glnx_throw_errno (error); } if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { - do - res = fchown (destination_dfd, - g_file_info_get_attribute_uint32 (source_info, "unix::uid"), - g_file_info_get_attribute_uint32 (source_info, "unix::gid")); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (fchown (destination_dfd, + g_file_info_get_attribute_uint32 (source_info, "unix::uid"), + g_file_info_get_attribute_uint32 (source_info, "unix::gid"))) < 0) return glnx_throw_errno (error); } @@ -758,10 +738,7 @@ checkout_tree_at_recurse (OstreeRepo *self, if (!did_exist) { const struct timespec times[2] = { { OSTREE_TIMESTAMP, UTIME_OMIT }, { OSTREE_TIMESTAMP, 0} }; - do - res = futimens (destination_dfd, times); - while (G_UNLIKELY (res == -1 && errno == EINTR)); - if (G_UNLIKELY (res == -1)) + if (TEMP_FAILURE_RETRY (futimens (destination_dfd, times)) < 0) return glnx_throw_errno (error); } From e60b9bc0492358d41e9646df608a40c3f6729f7c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 28 Apr 2017 09:32:35 -0400 Subject: [PATCH 30/94] ci: Add a context for testing flatpak This was part of the philosophy behind https://wiki.gnome.org/Initiatives/GnomeGoals/InstalledTests - libraries like libostree don't need to replicate everything in unit tests, we can use the tests from our dependencies directly too. We'll also get API break coverage testing too. Closes: #818 Approved by: jlebon --- .redhat-ci.yml | 23 +++++++++++++++++++++++ ci/flatpak.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100755 ci/flatpak.sh diff --git a/.redhat-ci.yml b/.redhat-ci.yml index e5604040..8b58f80c 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -135,3 +135,26 @@ tests: artifacts: - test-suite.log + +--- + +inherit: false +branches: + - master + - auto + - try + +context: f25-flatpak +required: false + +# This test case wants an "unprivileged container with bubblewrap", +# which we don't have right now; so just provision a VM and do a +# docker --privileged run. +host: + distro: fedora/25/atomic + +tests: + - docker run --rm --privileged -v $(pwd):/srv/code registry.fedoraproject.org/fedora:25 /bin/sh -c "cd /srv/code && ./ci/flatpak.sh" + +artifacts: + - test-suite.log diff --git a/ci/flatpak.sh b/ci/flatpak.sh new file mode 100755 index 00000000..f7edba3e --- /dev/null +++ b/ci/flatpak.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Build and run flatpak's unit tests using the just-built ostree for this PR. + +set -xeuo pipefail + +build() { + env NOCONFIGURE=1 ./autogen.sh + ./configure --prefix=/usr --libdir=/usr/lib64 "$@" + make -j 8 +} + +codedir=$(pwd) + +# Core prep +dnf -y install dnf-plugins-core +dnf install -y @buildsys-build +dnf install -y 'dnf-command(builddep)' + +# build+install ostree +dnf builddep -y ostree +build +make install +tmpd=$(mktemp -d) +cd ${tmpd} +# Frozen to a tag for now on general principle +git clone --recursive --depth=1 -b 0.9.3 https://github.com/flatpak/flatpak +cd flatpak +dnf builddep -y flatpak +# And runtime deps +dnf install -y flatpak && rpm -e flatpak +dnf install which attr fuse parallel # for the test suite +build +# We want to capture automake results from flatpak +cleanup() { + mv test-suite.log ${codedir} || true +} +trap cleanup EXIT +make -j 8 check From f74e52a3a0c441f8e56b796d99c492b1a0a2072c Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 28 Apr 2017 16:06:40 +0100 Subject: [PATCH 31/94] libostree: Rework OstreeAsyncProgress to use GVariants internally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OstreeAsyncProgress currently does some contortions to try and avoid allocating space for guints and guint64s (on 64-bit platforms), but this means it uses two GHashTables. A GHashTable allocates 8 buckets even when empty. Given that the largest usage of OstreeAsyncProgress in libostree puts 13 uints and 5 uint64s in it, this optimisation does not save significant (if any) memory. Instead, change OstreeAsyncProgress to store values internally as GVariants, and expose this with some new API: • ostree_async_progress_get_variant() • ostree_async_progress_set_variant() Each GVariant is allocated on the heap. As they are immutable, they are thread-safe once returned by a getter. The existing API continues to work as before, except in the case where a key is set/got as both a uint and a uint64 — there will now be a collision (and a GVariant type checking failure) whereas previously there was no collision. Nothing in OSTree uses OstreeAsyncProgress this way though. The new API can be used to share more complex data via the progress API. Signed-off-by: Philip Withnall Closes: #819 Approved by: cgwalters --- apidoc/ostree-sections.txt | 2 + src/libostree/libostree.sym | 8 +- src/libostree/ostree-async-progress.c | 129 ++++++++++++++------------ src/libostree/ostree-async-progress.h | 7 ++ 4 files changed, 81 insertions(+), 65 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index e6ed4c8a..5625a193 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -4,9 +4,11 @@ OstreeAsyncProgress ostree_async_progress_new ostree_async_progress_new_and_connect ostree_async_progress_get_status +ostree_async_progress_get_variant ostree_async_progress_get_uint ostree_async_progress_get_uint64 ostree_async_progress_set_status +ostree_async_progress_set_variant ostree_async_progress_set_uint ostree_async_progress_set_uint64 ostree_async_progress_finish diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index d9339bf2..0787d877 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -394,13 +394,11 @@ global: * NOTE NOTE NOTE */ -/* Uncomment when adding the first new symbol */ -/* -LIBOSTREE_2017.$NEWVERSION { +LIBOSTREE_2017.6 { global: - someostree_symbol_deleteme; + ostree_async_progress_get_variant; + ostree_async_progress_set_variant; } LIBOSTREE_2017.4; -*/ /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 0851fd86..68f4c834 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -22,6 +22,8 @@ #include "ostree-async-progress.h" +#include "libglnx.h" + /** * SECTION:ostree-async-progress * @title: Progress notification system for asynchronous operations @@ -39,12 +41,6 @@ * operation. */ -#if GLIB_SIZEOF_VOID_P == 8 -#define _OSTREE_HAVE_LP64 1 -#else -#define _OSTREE_HAVE_LP64 0 -#endif - enum { CHANGED, LAST_SIGNAL @@ -59,8 +55,7 @@ struct OstreeAsyncProgress GMutex lock; GMainContext *maincontext; GSource *idle_source; - GHashTable *uint_values; - GHashTable *uint64_values; + GHashTable *values; /* (element-type uint GVariant) */ gboolean dead; @@ -79,8 +74,7 @@ ostree_async_progress_finalize (GObject *object) g_mutex_clear (&self->lock); g_clear_pointer (&self->maincontext, g_main_context_unref); g_clear_pointer (&self->idle_source, g_source_unref); - g_hash_table_unref (self->uint_values); - g_hash_table_unref (self->uint64_values); + g_hash_table_unref (self->values); g_free (self->status); G_OBJECT_CLASS (ostree_async_progress_parent_class)->finalize (object); @@ -114,46 +108,53 @@ ostree_async_progress_init (OstreeAsyncProgress *self) { g_mutex_init (&self->lock); self->maincontext = g_main_context_ref_thread_default (); - self->uint_values = g_hash_table_new (NULL, NULL); -#if _OSTREE_HAVE_LP64 - self->uint64_values = g_hash_table_new (NULL, NULL); -#else - self->uint64_values = g_hash_table_new_full (NULL, NULL, - NULL, g_free); -#endif + self->values = g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify) g_variant_unref); +} + +/** + * ostree_async_progress_get_variant: + * @self: an #OstreeAsyncProgress + * @key: a key to look up + * + * Look up a key in the #OstreeAsyncProgress and return the #GVariant associated + * with it. The lookup is thread-safe. + * + * Returns: (transfer full) (nullable): value for the given @key, or %NULL if + * it was not set + * Since: 2017.6 + */ +GVariant * +ostree_async_progress_get_variant (OstreeAsyncProgress *self, + const char *key) +{ + GVariant *rval; + + g_return_val_if_fail (OSTREE_IS_ASYNC_PROGRESS (self), NULL); + g_return_val_if_fail (key != NULL, NULL); + + g_mutex_lock (&self->lock); + rval = g_hash_table_lookup (self->values, GUINT_TO_POINTER (g_quark_from_string (key))); + if (rval != NULL) + g_variant_ref (rval); + g_mutex_unlock (&self->lock); + + return rval; } guint ostree_async_progress_get_uint (OstreeAsyncProgress *self, const char *key) { - guint rval; - g_mutex_lock (&self->lock); - rval = GPOINTER_TO_UINT (g_hash_table_lookup (self->uint_values, - GUINT_TO_POINTER (g_quark_from_string (key)))); - g_mutex_unlock (&self->lock); - return rval; + g_autoptr(GVariant) rval = ostree_async_progress_get_variant (self, key); + return (rval != NULL) ? g_variant_get_uint32 (rval) : 0; } guint64 ostree_async_progress_get_uint64 (OstreeAsyncProgress *self, const char *key) { -#if _OSTREE_HAVE_LP64 - guint64 rval; - g_mutex_lock (&self->lock); - rval = (guint64) g_hash_table_lookup (self->uint64_values, GUINT_TO_POINTER (g_quark_from_string (key))); - g_mutex_unlock (&self->lock); - return rval; -#else - guint64 *rval; - g_mutex_lock (&self->lock); - rval = g_hash_table_lookup (self->uint64_values, (gpointer)g_quark_from_string (key)); - g_mutex_unlock (&self->lock); - if (rval) - return *rval; - return 0; -#endif + g_autoptr(GVariant) rval = ostree_async_progress_get_variant (self, key); + return (rval != NULL) ? g_variant_get_uint64 (rval) : 0; } static gboolean @@ -204,26 +205,45 @@ ostree_async_progress_get_status (OstreeAsyncProgress *self) return ret; } -static void -update_key (OstreeAsyncProgress *self, - GHashTable *hash, - const char *key, - gpointer value) +/** + * ostree_async_progress_set_variant: + * @self: an #OstreeAsyncProgress + * @key: a key to set + * @value: the value to assign to @key + * + * Assign a new @value to the given @key, replacing any existing value. The + * operation is thread-safe. @value may be a floating reference; + * g_variant_ref_sink() will be called on it. + * + * Any watchers of the #OstreeAsyncProgress will be notified of the change if + * @value differs from the existing value for @key. + * + * Since: 2017.6 + */ +void +ostree_async_progress_set_variant (OstreeAsyncProgress *self, + const char *key, + GVariant *value) { - gpointer orig_value; + GVariant *orig_value; + g_autoptr(GVariant) new_value = g_variant_ref_sink (value); gpointer qkey = GUINT_TO_POINTER (g_quark_from_string (key)); + g_return_if_fail (OSTREE_IS_ASYNC_PROGRESS (self)); + g_return_if_fail (key != NULL); + g_return_if_fail (value != NULL); + g_mutex_lock (&self->lock); if (self->dead) goto out; - if (g_hash_table_lookup_extended (hash, qkey, NULL, &orig_value)) + if (g_hash_table_lookup_extended (self->values, qkey, NULL, (gpointer *) &orig_value)) { - if (orig_value == value) + if (g_variant_equal (orig_value, new_value)) goto out; } - g_hash_table_replace (hash, qkey, value); + g_hash_table_replace (self->values, qkey, g_steal_pointer (&new_value)); ensure_callback_locked (self); out: @@ -235,7 +255,7 @@ ostree_async_progress_set_uint (OstreeAsyncProgress *self, const char *key, guint value) { - update_key (self, self->uint_values, key, GUINT_TO_POINTER (value)); + ostree_async_progress_set_variant (self, key, g_variant_new_uint32 (value)); } void @@ -243,18 +263,7 @@ ostree_async_progress_set_uint64 (OstreeAsyncProgress *self, const char *key, guint64 value) { - gpointer valuep; - -#if _OSTREE_HAVE_LP64 - valuep = (gpointer)value; -#else - { - guint64 *boxed = g_malloc (sizeof (guint64)); - *boxed = value; - valuep = boxed; - } -#endif - update_key (self, self->uint64_values, key, valuep); + ostree_async_progress_set_variant (self, key, g_variant_new_uint64 (value)); } /** diff --git a/src/libostree/ostree-async-progress.h b/src/libostree/ostree-async-progress.h index ae0e5faa..62bdcff8 100644 --- a/src/libostree/ostree-async-progress.h +++ b/src/libostree/ostree-async-progress.h @@ -59,6 +59,9 @@ guint ostree_async_progress_get_uint (OstreeAsyncProgress *self, _OSTREE_PUBLIC guint64 ostree_async_progress_get_uint64 (OstreeAsyncProgress *self, const char *key); +_OSTREE_PUBLIC +GVariant *ostree_async_progress_get_variant (OstreeAsyncProgress *self, + const char *key); _OSTREE_PUBLIC void ostree_async_progress_set_status (OstreeAsyncProgress *self, @@ -72,6 +75,10 @@ _OSTREE_PUBLIC void ostree_async_progress_set_uint64 (OstreeAsyncProgress *self, const char *key, guint64 value); +_OSTREE_PUBLIC +void ostree_async_progress_set_variant (OstreeAsyncProgress *self, + const char *key, + GVariant *value); _OSTREE_PUBLIC void ostree_async_progress_finish (OstreeAsyncProgress *self); From c27b66de80ee09077b770b8059a15e5c541e70e3 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 28 Apr 2017 16:18:55 +0100 Subject: [PATCH 32/94] libostree: Add multiple getter/setter support to OstreeAsyncProgress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OstreeAsyncProgress is thread-safe: it can have keys changed by one thread while another is getting the same keys (modulo some locking contention). However, the thread safety is done at the function call level: if some code calls an OstreeAsyncProgress getter several times, the key fetches are not atomic with respect to each other. In the case of contention on the lock, this can result in consumers of OstreeAsyncProgress data seeing an inconsistent state between the properties they query, which could result in progress reporting inaccuracies. In the uncontested case, this results in the OstreeAsyncProgress lock being locked and unlocked many times more than necessary. Try to improve this by adding new API, which supports getting and setting multiple keys atomically: • ostree_async_progress_get() • ostree_async_progress_set() The new API uses GVariants and varargs: keys are passed as a GVariantType string followed by arguments as for g_variant_new() or g_variant_get(), followed by the next key, etc. Signed-off-by: Philip Withnall Closes: #819 Approved by: cgwalters --- apidoc/ostree-sections.txt | 2 + src/libostree/libostree.sym | 2 + src/libostree/ostree-async-progress.c | 134 ++++++++++++++++++++++++++ src/libostree/ostree-async-progress.h | 8 ++ 4 files changed, 146 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 5625a193..adc2dfd7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -4,10 +4,12 @@ OstreeAsyncProgress ostree_async_progress_new ostree_async_progress_new_and_connect ostree_async_progress_get_status +ostree_async_progress_get ostree_async_progress_get_variant ostree_async_progress_get_uint ostree_async_progress_get_uint64 ostree_async_progress_set_status +ostree_async_progress_set ostree_async_progress_set_variant ostree_async_progress_set_uint ostree_async_progress_set_uint64 diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 0787d877..df74de53 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -396,6 +396,8 @@ global: LIBOSTREE_2017.6 { global: + ostree_async_progress_get; + ostree_async_progress_set; ostree_async_progress_get_variant; ostree_async_progress_set_variant; } LIBOSTREE_2017.4; diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 68f4c834..4927a9de 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -157,6 +157,68 @@ ostree_async_progress_get_uint64 (OstreeAsyncProgress *self, return (rval != NULL) ? g_variant_get_uint64 (rval) : 0; } +/** + * ostree_async_progress_get: + * @self: an #OstreeAsyncProgress + * @...: key name, format string, #GVariant return locations, …, followed by %NULL + * + * Get the values corresponding to zero or more keys from the + * #OstreeAsyncProgress. Each key is specified in @... as the key name, followed + * by a #GVariant format string, followed by the necessary arguments for that + * format string, just as for g_variant_get(). After those arguments is the + * next key name. The varargs list must be %NULL-terminated. + * + * Each format string must make deep copies of its value, as the values stored + * in the #OstreeAsyncProgress may be freed from another thread after this + * function returns. + * + * This operation is thread-safe, and all the keys are queried atomically. + * + * |[ + * guint32 outstanding_fetches; + * guint64 bytes_received; + * g_autofree gchar *status = NULL; + * g_autoptr(GVariant) refs_variant = NULL; + * + * ostree_async_progress_get (progress, + * "outstanding-fetches", "u", &outstanding_fetches, + * "bytes-received", "t", &bytes_received, + * "status", "s", &status, + * "refs", "@a{ss}", &refs_variant, + * NULL); + * ]| + * + * Since: 2017.6 + */ +void +ostree_async_progress_get (OstreeAsyncProgress *self, + ...) +{ + va_list ap; + const char *key, *format_string; + + g_mutex_lock (&self->lock); + va_start (ap, self); + + for (key = va_arg (ap, const char *), format_string = va_arg (ap, const char *); + key != NULL; + key = va_arg (ap, const char *), format_string = va_arg (ap, const char *)) + { + GVariant *variant; + + g_assert (format_string != NULL); + + variant = g_hash_table_lookup (self->values, GUINT_TO_POINTER (g_quark_from_string (key))); + g_assert (variant != NULL); + g_assert (g_variant_check_format_string (variant, format_string, TRUE)); + + g_variant_get_va (variant, format_string, NULL, &ap); + } + + va_end (ap); + g_mutex_unlock (&self->lock); +} + static gboolean idle_invoke_async_progress (gpointer user_data) { @@ -205,6 +267,78 @@ ostree_async_progress_get_status (OstreeAsyncProgress *self) return ret; } +/** + * ostree_async_progress_set: + * @self: an #OstreeAsyncProgress + * @...: key name, format string, #GVariant parameters, …, followed by %NULL + * + * Set the values for zero or more keys in the #OstreeAsyncProgress. Each key is + * specified in @... as the key name, followed by a #GVariant format string, + * followed by the necessary arguments for that format string, just as for + * g_variant_new(). After those arguments is the next key name. The varargs list + * must be %NULL-terminated. + * + * g_variant_ref_sink() will be called as appropriate on the #GVariant + * parameters, so they may be floating. + * + * This operation is thread-safe, and all the keys are set atomically. + * + * |[ + * guint32 outstanding_fetches = 15; + * guint64 bytes_received = 1000; + * + * ostree_async_progress_set (progress, + * "outstanding-fetches", "u", outstanding_fetches, + * "bytes-received", "t", bytes_received, + * "status", "s", "Updated status", + * "refs", "@a{ss}", g_variant_new_parsed ("@a{ss} {}"), + * NULL); + * ]| + * + * Since: 2017.6 + */ +void +ostree_async_progress_set (OstreeAsyncProgress *self, + ...) +{ + va_list ap; + const char *key, *format_string; + gboolean changed; + + g_mutex_lock (&self->lock); + + if (self->dead) + goto out; + + va_start (ap, self); + + for (key = va_arg (ap, const char *), format_string = va_arg (ap, const char *); + key != NULL; + key = va_arg (ap, const char *), format_string = va_arg (ap, const char *)) + { + GVariant *orig_value; + g_autoptr(GVariant) new_value = NULL; + gpointer qkey = GUINT_TO_POINTER (g_quark_from_string (key)); + + new_value = g_variant_ref_sink (g_variant_new_va (format_string, NULL, &ap)); + + if (g_hash_table_lookup_extended (self->values, qkey, NULL, (gpointer *) &orig_value) && + g_variant_equal (orig_value, new_value)) + continue; + + g_hash_table_replace (self->values, qkey, g_steal_pointer (&new_value)); + changed = TRUE; + } + + va_end (ap); + + if (changed) + ensure_callback_locked (self); + +out: + g_mutex_unlock (&self->lock); +} + /** * ostree_async_progress_set_variant: * @self: an #OstreeAsyncProgress diff --git a/src/libostree/ostree-async-progress.h b/src/libostree/ostree-async-progress.h index 62bdcff8..55ac591c 100644 --- a/src/libostree/ostree-async-progress.h +++ b/src/libostree/ostree-async-progress.h @@ -53,6 +53,10 @@ OstreeAsyncProgress *ostree_async_progress_new_and_connect (void (*changed) (Ost _OSTREE_PUBLIC char *ostree_async_progress_get_status (OstreeAsyncProgress *self); +_OSTREE_PUBLIC +void ostree_async_progress_get (OstreeAsyncProgress *self, + ...) G_GNUC_NULL_TERMINATED; + _OSTREE_PUBLIC guint ostree_async_progress_get_uint (OstreeAsyncProgress *self, const char *key); @@ -67,6 +71,10 @@ _OSTREE_PUBLIC void ostree_async_progress_set_status (OstreeAsyncProgress *self, const char *status); +_OSTREE_PUBLIC +void ostree_async_progress_set (OstreeAsyncProgress *self, + ...) G_GNUC_NULL_TERMINATED; + _OSTREE_PUBLIC void ostree_async_progress_set_uint (OstreeAsyncProgress *self, const char *key, From cdf876101be1b61202489c40c8f5b2749a02577d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 28 Apr 2017 16:24:00 +0100 Subject: [PATCH 33/94] src: Port to new OstreeAsyncProgress atomic API This will eliminate most of the potential races in progress reporting. ostree_repo_pull_default_console_progress_changed() still calls three getters, so there may still be races there, however. Signed-off-by: Philip Withnall Closes: #819 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 60 ++++++++++++++++---------------- src/libostree/ostree-repo.c | 44 ++++++++++++++--------- src/ostree/ot-builtin-pull.c | 21 ++++++----- 3 files changed, 69 insertions(+), 56 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 3f5b771c..f7dbd79e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -211,36 +211,36 @@ update_progress (gpointer user_data) n_scanned_metadata = pull_data->n_scanned_metadata; start_time = pull_data->start_time; - ostree_async_progress_set_uint (pull_data->progress, "outstanding-fetches", outstanding_fetches); - ostree_async_progress_set_uint (pull_data->progress, "outstanding-writes", outstanding_writes); - ostree_async_progress_set_uint (pull_data->progress, "fetched", fetched); - ostree_async_progress_set_uint (pull_data->progress, "requested", requested); - ostree_async_progress_set_uint (pull_data->progress, "scanning", g_queue_is_empty (&pull_data->scan_object_queue) ? 0 : 1); - ostree_async_progress_set_uint (pull_data->progress, "scanned-metadata", n_scanned_metadata); - ostree_async_progress_set_uint64 (pull_data->progress, "bytes-transferred", bytes_transferred); - ostree_async_progress_set_uint64 (pull_data->progress, "start-time", start_time); - - /* Deltas */ - ostree_async_progress_set_uint (pull_data->progress, "fetched-delta-parts", - pull_data->n_fetched_deltaparts); - ostree_async_progress_set_uint (pull_data->progress, "total-delta-parts", - pull_data->n_total_deltaparts); - ostree_async_progress_set_uint (pull_data->progress, "fetched-delta-fallbacks", - pull_data->n_fetched_deltapart_fallbacks); - ostree_async_progress_set_uint (pull_data->progress, "total-delta-fallbacks", - pull_data->n_total_delta_fallbacks); - ostree_async_progress_set_uint64 (pull_data->progress, "fetched-delta-part-size", - pull_data->fetched_deltapart_size); - ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size", - pull_data->total_deltapart_size); - ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize", - pull_data->total_deltapart_usize); - ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks", - pull_data->static_delta_superblocks->len); - - /* We fetch metadata before content. These allow us to report metadata fetch progress specifically. */ - ostree_async_progress_set_uint (pull_data->progress, "outstanding-metadata-fetches", pull_data->n_outstanding_metadata_fetches); - ostree_async_progress_set_uint (pull_data->progress, "metadata-fetched", pull_data->n_fetched_metadata); + ostree_async_progress_set (pull_data->progress, + "outstanding-fetches", "u", outstanding_fetches, + "outstanding-writes", "u", outstanding_writes, + "fetched", "u", fetched, + "requested", "u", requested, + "scanning", "u", g_queue_is_empty (&pull_data->scan_object_queue) ? 0 : 1, + "scanned-metadata", "u", n_scanned_metadata, + "bytes-transferred", "t", bytes_transferred, + "start-time", "t", start_time, + /* Deltas */ + "fetched-delta-parts", + "u", pull_data->n_fetched_deltaparts, + "total-delta-parts", + "u", pull_data->n_total_deltaparts, + "fetched-delta-fallbacks", + "u", pull_data->n_fetched_deltapart_fallbacks, + "total-delta-fallbacks", + "u", pull_data->n_total_delta_fallbacks, + "fetched-delta-part-size", + "t", pull_data->fetched_deltapart_size, + "total-delta-part-size", + "t", pull_data->total_deltapart_size, + "total-delta-part-usize", + "t", pull_data->total_deltapart_usize, + "total-delta-superblocks", + "u", pull_data->static_delta_superblocks->len, + /* We fetch metadata before content. These allow us to report metadata fetch progress specifically. */ + "outstanding-metadata-fetches", "u", pull_data->n_outstanding_metadata_fetches, + "metadata-fetched", "u", pull_data->n_fetched_metadata, + NULL); ostree_async_progress_set_status (pull_data->progress, NULL); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index ec2d8c83..8e2b1c21 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3749,15 +3749,18 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress buf = g_string_new (""); status = ostree_async_progress_get_status (progress); - outstanding_fetches = ostree_async_progress_get_uint (progress, "outstanding-fetches"); - outstanding_metadata_fetches = ostree_async_progress_get_uint (progress, "outstanding-metadata-fetches"); - outstanding_writes = ostree_async_progress_get_uint (progress, "outstanding-writes"); - scanning = ostree_async_progress_get_uint (progress, "scanning") == 1; - n_scanned_metadata = ostree_async_progress_get_uint (progress, "scanned-metadata"); - fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); - total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); - fetched_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "fetched-delta-fallbacks"); - total_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "total-delta-fallbacks"); + + ostree_async_progress_get (progress, + "outstanding-fetches", "u", &outstanding_fetches, + "outstanding-metadata-fetches", "u", &outstanding_metadata_fetches, + "outstanding-writes", "u", &outstanding_writes, + "scanning", "u", &scanning, + "scanned-metadata", "u", &n_scanned_metadata, + "fetched-delta-parts", "u", &fetched_delta_parts, + "total-delta-parts", "u", &total_delta_parts, + "fetched-delta-fallbacks", "u", &fetched_delta_part_fallbacks, + "total-delta-fallbacks", "u", &total_delta_part_fallbacks, + NULL); if (status) { @@ -3765,18 +3768,25 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress } else if (outstanding_fetches) { - guint64 bytes_transferred = ostree_async_progress_get_uint64 (progress, "bytes-transferred"); - guint fetched = ostree_async_progress_get_uint (progress, "fetched"); - guint metadata_fetched = ostree_async_progress_get_uint (progress, "metadata-fetched"); - guint requested = ostree_async_progress_get_uint (progress, "requested"); - guint64 start_time = ostree_async_progress_get_uint64 (progress, "start-time"); - guint64 total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); + guint64 bytes_transferred, start_time, total_delta_part_size; + guint fetched, metadata_fetched, requested; guint64 current_time = g_get_monotonic_time (); - g_autofree char *formatted_bytes_transferred = - g_format_size_full (bytes_transferred, 0); + g_autofree char *formatted_bytes_transferred = NULL; g_autofree char *formatted_bytes_sec = NULL; guint64 bytes_sec; + /* Note: This is not atomic wrt the above getter call. */ + ostree_async_progress_get (progress, + "bytes-transferred", "t", &bytes_transferred, + "fetched", "u", &fetched, + "metadata-fetched", "u", &metadata_fetched, + "requested", "u", &requested, + "start-time", "t", &start_time, + "total-delta-part-size", "t", &total_delta_part_size, + NULL); + + formatted_bytes_transferred = g_format_size_full (bytes_transferred, 0); + /* Ignore the first second, or when we haven't transferred any * data, since those could cause divide by zero below. */ diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index a7826542..08f0028a 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -89,20 +89,23 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, g_assert (!printed_console_progress); printed_console_progress = TRUE; - /* Number of parts */ - fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); - total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); - fetched_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "fetched-delta-fallbacks"); - total_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "total-delta-fallbacks"); + ostree_async_progress_get (progress, + /* Number of parts */ + "fetched-delta-parts", "u", &fetched_delta_parts, + "total-delta-parts", "u", &total_delta_parts, + "fetched-delta-fallbacks", "u", &fetched_delta_part_fallbacks, + "total-delta-fallbacks", "u", &total_delta_part_fallbacks, + /* Size variables */ + "fetched-delta-part-size", "t", &fetched_delta_part_size, + "total-delta-part-size", "t", &total_delta_part_size, + "total-delta-part-usize", "t", &total_delta_part_usize, + NULL); + /* Fold the count of deltaparts + fallbacks for simplicity; if changing this, * please change ostree_repo_pull_default_console_progress_changed() first. */ fetched_delta_parts += fetched_delta_part_fallbacks; total_delta_parts += total_delta_part_fallbacks; - /* Size variables */ - fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size"); - total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); - total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize"); buf = g_string_new (""); From ce83abb868a6ecff5e8563532e76e742579e5064 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 28 Apr 2017 19:04:29 +0100 Subject: [PATCH 34/94] libostree: Allow OstreeAsyncProgress:status to be set atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework how the status is handled in OstreeAsyncProgress so that it’s now a well-known key in the hash table. This means that it can be retrieved and set atomically with other keys using ostree_async_progress_[get|set](). The behaviour of ostree_async_progress_[get|set]_status() is preserved, with the caveat that `status` can now also be accessed using the other API on OstreeAsyncProgress, and has to be accessed with the right GVariant type. Internally, a NULL status is represented by an empty status string (since ostree_async_progress_[get|set]_variant() deliberately don’t allow NULL variants to be set against keys, since that would break the ostree_async_progress_get() API). Signed-off-by: Philip Withnall Closes: #819 Approved by: cgwalters --- src/libostree/ostree-async-progress.c | 53 +++++++++++++++++++-------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 4927a9de..95556814 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -39,6 +39,11 @@ * handles thread safety, ensuring that the progress change * notification occurs in the thread-default context of the calling * operation. + * + * The ostree_async_progress_get_status() and ostree_async_progress_set_status() + * methods get and set a well-known `status` key of type %G_VARIANT_TYPE_STRING. + * This key may be accessed using the other #OstreeAsyncProgress methods, but it + * must always have the correct type. */ enum { @@ -58,8 +63,6 @@ struct OstreeAsyncProgress GHashTable *values; /* (element-type uint GVariant) */ gboolean dead; - - char *status; }; G_DEFINE_TYPE (OstreeAsyncProgress, ostree_async_progress, G_TYPE_OBJECT) @@ -75,7 +78,6 @@ ostree_async_progress_finalize (GObject *object) g_clear_pointer (&self->maincontext, g_main_context_unref); g_clear_pointer (&self->idle_source, g_source_unref); g_hash_table_unref (self->values); - g_free (self->status); G_OBJECT_CLASS (ostree_async_progress_parent_class)->finalize (object); } @@ -243,28 +245,47 @@ ensure_callback_locked (OstreeAsyncProgress *self) g_source_attach (self->idle_source, self->maincontext); } +/** + * ostree_async_progress_set_status: + * @self: an #OstreeAsyncProgress + * @status: (nullable): new status string, or %NULL to clear the status + * + * Set the human-readable status string for the #OstreeAsyncProgress. This + * operation is thread-safe. %NULL may be passed to clear the status. + * + * This is a convenience function to set the well-known `status` key. + * + * Since: 2017.6 + */ void ostree_async_progress_set_status (OstreeAsyncProgress *self, const char *status) { - g_mutex_lock (&self->lock); - if (!self->dead) - { - g_free (self->status); - self->status = g_strdup (status); - ensure_callback_locked (self); - } - g_mutex_unlock (&self->lock); + ostree_async_progress_set_variant (self, "status", + g_variant_new_string ((status != NULL) ? status : "")); } +/** + * ostree_async_progress_get_status: + * @self: an #OstreeAsyncProgress + * + * Get the human-readable status string from the #OstreeAsyncProgress. This + * operation is thread-safe. The retuned value may be %NULL if no status is + * set. + * + * This is a convenience function to get the well-known `status` key. + * + * Returns: (transfer full) (nullable): the current status, or %NULL if none is set + * Since: 2017.6 + */ char * ostree_async_progress_get_status (OstreeAsyncProgress *self) { - char *ret; - g_mutex_lock (&self->lock); - ret = g_strdup (self->status); - g_mutex_unlock (&self->lock); - return ret; + g_autoptr(GVariant) rval = ostree_async_progress_get_variant (self, "status"); + const gchar *status = (rval != NULL) ? g_variant_get_string (rval, NULL) : NULL; + if (status != NULL && *status == '\0') + status = NULL; + return g_strdup (status); } /** From cbe3989b2bd0ec93f9878c2d9a79fcb72d477d11 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 28 Apr 2017 19:32:17 +0100 Subject: [PATCH 35/94] libostree: Get and set OstreeAsyncProgress:status atomically Use the new well-known `status` key for OstreeAsyncProgress to get and set the status atomically with other keys in an OstreeAsyncProgress instance. Signed-off-by: Philip Withnall Closes: #819 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 4 ++-- src/libostree/ostree-repo.c | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f7dbd79e..c6b63669 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -240,10 +240,10 @@ update_progress (gpointer user_data) /* We fetch metadata before content. These allow us to report metadata fetch progress specifically. */ "outstanding-metadata-fetches", "u", pull_data->n_outstanding_metadata_fetches, "metadata-fetched", "u", pull_data->n_fetched_metadata, + /* Overall status. */ + "status", "s", "", NULL); - ostree_async_progress_set_status (pull_data->progress, NULL); - if (pull_data->dry_run) pull_data->dry_run_emitted_progress = TRUE; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8e2b1c21..1bb4d3c3 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3748,8 +3748,6 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress buf = g_string_new (""); - status = ostree_async_progress_get_status (progress); - ostree_async_progress_get (progress, "outstanding-fetches", "u", &outstanding_fetches, "outstanding-metadata-fetches", "u", &outstanding_metadata_fetches, @@ -3760,9 +3758,10 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress "total-delta-parts", "u", &total_delta_parts, "fetched-delta-fallbacks", "u", &fetched_delta_part_fallbacks, "total-delta-fallbacks", "u", &total_delta_part_fallbacks, + "status", "s", &status, NULL); - if (status) + if (*status != '\0') { g_string_append (buf, status); } From dae6cfba56efab5977786e9a519e8951a2e182db Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 28 Apr 2017 16:58:34 -0400 Subject: [PATCH 36/94] ci: Fix flatpak test pkg install Closes: #822 Approved by: jlebon --- ci/flatpak.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/flatpak.sh b/ci/flatpak.sh index f7edba3e..fee738aa 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -28,7 +28,7 @@ cd flatpak dnf builddep -y flatpak # And runtime deps dnf install -y flatpak && rpm -e flatpak -dnf install which attr fuse parallel # for the test suite +dnf install -y which attr fuse parallel # for the test suite build # We want to capture automake results from flatpak cleanup() { From 8d8f06f21b3f316c927cd85b573c2e331534af4a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 28 Apr 2017 18:02:22 -0400 Subject: [PATCH 37/94] checkout: Dedup calls to memcache ref Minor, but I realized `checkout_tree_at()` is a better place to do common setup before checkout. Prep for https://github.com/ostreedev/ostree/pull/813 Closes: #823 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index fa29699b..36be420b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -777,6 +777,11 @@ checkout_tree_at (OstreeRepo *self, g_assert (options->force_copy); } + /* Cache any directory metadata we read during this operation; + * see commit b7afe91e21143d7abb0adde440683a52712aa246 + */ + g_auto(OstreeRepoMemoryCacheRef) memcache_ref; + _ostree_repo_memory_cache_ref_init (&memcache_ref, self); return checkout_tree_at_recurse (self, options, &state, destination_parent_fd, destination_name, source, source_info, @@ -829,8 +834,6 @@ ostree_repo_checkout_tree (OstreeRepo *self, options.enable_uncompressed_cache = TRUE; canonicalize_options (self, &options); - g_auto(OstreeRepoMemoryCacheRef) memcache_ref; - _ostree_repo_memory_cache_ref_init (&memcache_ref, self); return checkout_tree_at (self, &options, AT_FDCWD, gs_file_get_path_cached (destination), source, source_info, @@ -948,8 +951,6 @@ ostree_repo_checkout_at (OstreeRepo *self, if (!target_info) return FALSE; - g_auto(OstreeRepoMemoryCacheRef) memcache_ref; - _ostree_repo_memory_cache_ref_init (&memcache_ref, self); if (!checkout_tree_at (self, options, destination_dfd, destination_path, From 4f80548454dba1342849caf5b568cb4e34883ba4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 28 Apr 2017 16:00:49 -0400 Subject: [PATCH 38/94] repo: Delete the last use of GFile tmp_dir The keyring isn't large, so let's just fall back to copying it rather than requiring `renameat()`. Prep for `ostree_repo_open_at()`. Closes: #821 Approved by: jlebon --- src/libostree/ostree-gpg-verifier.c | 2 +- src/libostree/ostree-repo-private.h | 1 - src/libostree/ostree-repo.c | 37 ++++++++--------------------- src/libotutil/ot-gpg-utils.c | 6 +---- src/libotutil/ot-gpg-utils.h | 1 - 5 files changed, 12 insertions(+), 35 deletions(-) diff --git a/src/libostree/ostree-gpg-verifier.c b/src/libostree/ostree-gpg-verifier.c index eda05e9a..59028821 100644 --- a/src/libostree/ostree-gpg-verifier.c +++ b/src/libostree/ostree-gpg-verifier.c @@ -116,7 +116,7 @@ _ostree_gpg_verifier_check_signature (OstreeGpgVerifier *self, if (result == NULL) goto out; - if (!ot_gpgme_ctx_tmp_home_dir (result->context, NULL, + if (!ot_gpgme_ctx_tmp_home_dir (result->context, &tmp_dir, &target_stream, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 74f032d1..94330226 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -78,7 +78,6 @@ struct OstreeRepo { GFile *repodir; int repo_dir_fd; - GFile *tmp_dir; int tmp_dir_fd; int cache_dir_fd; char *cache_dir; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1bb4d3c3..1104b91a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -517,7 +517,6 @@ ostree_repo_finalize (GObject *object) (void) close (self->commit_stagedir_fd); g_free (self->commit_stagedir_name); glnx_release_lock_file (&self->commit_stagedir_lock); - g_clear_object (&self->tmp_dir); if (self->tmp_dir_fd != -1) (void) close (self->tmp_dir_fd); if (self->cache_dir_fd != -1) @@ -605,8 +604,6 @@ ostree_repo_constructed (GObject *object) g_assert (self->repodir != NULL); - self->tmp_dir = g_file_resolve_relative_path (self->repodir, "tmp"); - /* Ensure the "sysroot-path" property is set. */ if (self->sysroot_dir == NULL) self->sysroot_dir = g_object_ref (_ostree_get_default_sysroot_path ()); @@ -1401,7 +1398,6 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, ot_auto_gpgme_data gpgme_data_t data_buffer = NULL; gpgme_import_result_t import_result; gpgme_import_status_t import_status; - const char *tmp_dir = NULL; g_autofree char *source_tmp_dir = NULL; g_autofree char *target_tmp_dir = NULL; glnx_fd_close int target_temp_fd = -1; @@ -1409,6 +1405,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, struct stat stbuf; gpgme_error_t gpg_error; gboolean ret = FALSE; + const GLnxFileCopyFlags copyflags = self->disable_xattrs ? GLNX_FILE_COPY_NOXATTRS : 0; g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); @@ -1419,17 +1416,6 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, if (remote == NULL) goto out; - /* Use OstreeRepo's "tmp" directory so the keyring files remain - * under one mount point. Necessary for renameat() below. */ - - /* XXX This produces a path under "/proc/self/fd/" which won't - * work in a child process so I had to resort to the GFile. - * I was trying to avoid the GFile so we can get rid of it. - * - * tmp_dir = glnx_fdrel_abspath (self->repo_dir_fd, "tmp"); - */ - tmp_dir = gs_file_get_path_cached (self->tmp_dir); - /* Prepare the source GPGME context. If reading GPG keys from an input * stream, point the OpenPGP engine at a temporary directory and import * the keys to a new pubring.gpg file. If the key data format is ASCII @@ -1443,7 +1429,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, { data_buffer = ot_gpgme_data_input (source_stream); - if (!ot_gpgme_ctx_tmp_home_dir (source_context, tmp_dir, &source_tmp_dir, + if (!ot_gpgme_ctx_tmp_home_dir (source_context, &source_tmp_dir, NULL, cancellable, error)) { g_prefix_error (error, "Unable to configure context: "); @@ -1526,7 +1512,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, goto out; /* No need for an output stream since we copy in a pubring.gpg. */ - if (!ot_gpgme_ctx_tmp_home_dir (target_context, tmp_dir, &target_tmp_dir, + if (!ot_gpgme_ctx_tmp_home_dir (target_context, &target_tmp_dir, NULL, cancellable, error)) { g_prefix_error (error, "Unable to configure context: "); @@ -1541,10 +1527,9 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, if (fstatat (self->repo_dir_fd, remote->keyring, &stbuf, AT_SYMLINK_NOFOLLOW) == 0) { - GLnxFileCopyFlags copyflags = self->disable_xattrs ? GLNX_FILE_COPY_NOXATTRS : 0; if (!glnx_file_copy_at (self->repo_dir_fd, remote->keyring, - &stbuf, target_temp_fd, "pubring.gpg", copyflags, - cancellable, error)) + &stbuf, target_temp_fd, "pubring.gpg", + copyflags, cancellable, error)) { g_prefix_error (error, "Unable to copy remote's keyring: "); goto out; @@ -1626,13 +1611,11 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, /* Import successful; replace the remote's old keyring with the * updated keyring in the target context's temporary directory. */ - - if (renameat (target_temp_fd, "pubring.gpg", - self->repo_dir_fd, remote->keyring) == -1) - { - glnx_set_prefix_error_from_errno (error, "%s", "Unable to rename keyring"); - goto out; - } + if (!glnx_file_copy_at (target_temp_fd, "pubring.gpg", NULL, + self->repo_dir_fd, remote->keyring, + copyflags | GLNX_FILE_COPY_OVERWRITE, + cancellable, error)) + goto out; if (out_imported != NULL) *out_imported = (guint) import_result->imported; diff --git a/src/libotutil/ot-gpg-utils.c b/src/libotutil/ot-gpg-utils.c index b71f4845..001daa0a 100644 --- a/src/libotutil/ot-gpg-utils.c +++ b/src/libotutil/ot-gpg-utils.c @@ -67,7 +67,6 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, gboolean ot_gpgme_ctx_tmp_home_dir (gpgme_ctx_t gpgme_ctx, - const char *tmp_dir, char **out_tmp_home_dir, GOutputStream **out_pubring_stream, GCancellable *cancellable, @@ -85,10 +84,7 @@ ot_gpgme_ctx_tmp_home_dir (gpgme_ctx_t gpgme_ctx, * and hand the caller an open output stream to concatenate necessary * keyring files. */ - if (tmp_dir == NULL) - tmp_dir = g_get_tmp_dir (); - - tmp_home_dir = g_build_filename (tmp_dir, "ostree-gpg-XXXXXX", NULL); + tmp_home_dir = g_build_filename (g_get_tmp_dir (), "ostree-gpg-XXXXXX", NULL); if (mkdtemp (tmp_home_dir) == NULL) { diff --git a/src/libotutil/ot-gpg-utils.h b/src/libotutil/ot-gpg-utils.h index c2337f7b..184a8d64 100644 --- a/src/libotutil/ot-gpg-utils.h +++ b/src/libotutil/ot-gpg-utils.h @@ -34,7 +34,6 @@ GLNX_DEFINE_CLEANUP_FUNCTION0(gpgme_ctx_t, ot_cleanup_gpgme_ctx, gpgme_release) void ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, GError **error); gboolean ot_gpgme_ctx_tmp_home_dir (gpgme_ctx_t gpgme_ctx, - const char *tmp_dir, char **out_tmp_home_dir, GOutputStream **out_pubring_stream, GCancellable *cancellable, From a1c866ed52134bac2aff79cd95e7558edd788032 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Wed, 3 May 2017 14:58:07 +0100 Subject: [PATCH 39/94] libostree: Fix a typo in docs for ostree_repo_pull_with_options() Signed-off-by: Philip Withnall Closes: #828 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c6b63669..2cfa2e86 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2725,7 +2725,7 @@ initiate_request (OtPullData *pull_data, * * override-commit-ids (as): Array of specific commit IDs to fetch for refs * * dry-run (b): Only print information on what will be downloaded (requires static deltas) * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options - * * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction. + * * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do multiple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid */ From 4c731165bb945bff834ed6a4129c5305d841073a Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 2 May 2017 22:33:04 +0100 Subject: [PATCH 40/94] libostree: Add missing checks for invalid timestamps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit g_date_time_new_from_unix_utc() will not always return a valid GDateTime — if the input timestamp is too big, GDateTime cannot represent it, and the constructor returns NULL. Add some missing checks for these situations. We don’t ever expect timestamps to be this big, but they could be as a result of corruption or a malicious repository. Signed-off-by: Philip Withnall Closes: #825 Approved by: cgwalters --- src/libostree/ostree-gpg-verify-result.c | 16 ++++++++++++++++ src/libostree/ostree-sysroot-upgrader.c | 11 +++++++++-- src/ostree/ot-remote-cookie-util.c | 8 ++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-gpg-verify-result.c b/src/libostree/ostree-gpg-verify-result.c index 73fbfeed..cd709b7c 100644 --- a/src/libostree/ostree-gpg-verify-result.c +++ b/src/libostree/ostree-gpg-verify-result.c @@ -562,6 +562,14 @@ ostree_gpg_verify_result_describe_variant (GVariant *variant, key_id = (len > 16) ? fingerprint + len - 16 : fingerprint; date_time_utc = g_date_time_new_from_unix_utc (timestamp); + if (date_time_utc == NULL) + { + g_string_append_printf (output_buffer, + "Can't check signature: timestamp %" G_GINT64_FORMAT " is invalid\n", + timestamp); + return; + } + date_time_local = g_date_time_to_local (date_time_utc); formatted_date_time = g_date_time_format (date_time_local, "%c"); @@ -606,6 +614,14 @@ ostree_gpg_verify_result_describe_variant (GVariant *variant, if (exp_timestamp > 0) { date_time_utc = g_date_time_new_from_unix_utc (exp_timestamp); + if (date_time_utc == NULL) + { + g_string_append_printf (output_buffer, + "Signature expiry timestamp (%" G_GINT64_FORMAT ") is invalid\n", + exp_timestamp); + return; + } + date_time_local = g_date_time_to_local (date_time_utc); formatted_date_time = g_date_time_format (date_time_local, "%c"); diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 4e7c8bf3..11d9706c 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -470,8 +470,15 @@ ostree_sysroot_upgrader_check_timestamps (OstreeRepo *repo, g_autofree char *old_ts_str = NULL; g_autofree char *new_ts_str = NULL; - g_assert (old_ts); - g_assert (new_ts); + if (old_ts == NULL || new_ts == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Upgrade target revision '%s' timestamp (%" G_GINT64_FORMAT ") or current revision '%s' timestamp (%" G_GINT64_FORMAT ") is invalid", + to_rev, ostree_commit_get_timestamp (new_commit), + from_rev, ostree_commit_get_timestamp (old_commit)); + goto out; + } + old_ts_str = g_date_time_format (old_ts, "%c"); new_ts_str = g_date_time_format (new_ts, "%c"); g_date_time_unref (old_ts); diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index a96038aa..e3ca9eac 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -298,14 +298,18 @@ ot_list_cookies_at (int dfd, const char *jar_path, GError **error) while (ot_parse_cookies_next (parser)) { g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); - g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); + g_autofree char *expires_str = NULL; + + if (expires != NULL) + expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); g_print ("--\n"); g_print ("Domain: %s\n", parser->domain); g_print ("Path: %s\n", parser->path); g_print ("Name: %s\n", parser->name); g_print ("Secure: %s\n", parser->secure); - g_print ("Expires: %s\n", expires_str); + if (expires_str != NULL) + g_print ("Expires: %s\n", expires_str); g_print ("Value: %s\n", parser->value); } #else From 59897f2b8484e322a77f23cc0d532dbd3de6aac6 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Wed, 3 May 2017 11:31:58 -0500 Subject: [PATCH 41/94] pull: Fix crash specifying override URL in summary fetch The summary URL override is looked up with "&s", which directly exchanges the data to a pointer without allocation. This was causing a segfault calling ostree_repo_remote_fetch_summary_with_options from pygobject. Closes: #829 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 2cfa2e86..35598eca 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2459,7 +2459,7 @@ repo_remote_fetch_summary (OstreeRepo *self, g_autoptr(GMainContext) mainctx = NULL; gboolean ret = FALSE; gboolean from_cache = FALSE; - g_autofree char *url_override = NULL; + const char *url_override = NULL; g_autoptr(GPtrArray) mirrorlist = NULL; if (options) From f1da7ec300d2827460d138a0f316ad6b77501336 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 5 May 2017 15:16:46 +0100 Subject: [PATCH 42/94] libostree: Fix potential use of uninitialised memory in progress API Signed-off-by: Philip Withnall Closes: #835 Approved by: cgwalters --- src/libostree/ostree-async-progress.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-async-progress.c b/src/libostree/ostree-async-progress.c index 95556814..b879559e 100644 --- a/src/libostree/ostree-async-progress.c +++ b/src/libostree/ostree-async-progress.c @@ -331,6 +331,8 @@ ostree_async_progress_set (OstreeAsyncProgress *self, if (self->dead) goto out; + changed = FALSE; + va_start (ap, self); for (key = va_arg (ap, const char *), format_string = va_arg (ap, const char *); From bf9772f2316a567566c20979e5247b2928287253 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 5 May 2017 15:17:13 +0100 Subject: [PATCH 43/94] libostree: Ensure progress keys are all always set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If one of the progress keys is set in a pull operation, a ::changed signal is emitted on the progress object, and the callback for that could query any of the progress keys — so they all need to be set, otherwise we get an assertion failure in ostree_async_progress_get() due to a named key not existing. Spotted by Dan Nicholson in PR #819. Signed-off-by: Philip Withnall Closes: #835 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 35598eca..12f93ab0 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3477,6 +3477,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, guint shift; GString *buf = g_string_new (""); + /* Ensure the rest of the progress keys are set appropriately. */ + update_progress (pull_data); + if (bytes_transferred < 1024) shift = 1; else From 712bf219141bdb13c42629d518578b62906fcaa7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 5 May 2017 10:44:49 -0400 Subject: [PATCH 44/94] tree-wide: Convert to using autoptr(GString) vs g_string_free(...,TRUE) If we're freeing the segment, it's basically always better to use `autoptr()`. Fewer lines, more reliable, etc. Noticed an instance of this in the pull code while reviewing a different PR, decided to do a grep for it and fix it tree wide. Closes: #836 Approved by: pwithnall --- src/libostree/ostree-bootloader-grub2.c | 4 +--- src/libostree/ostree-repo-commit.c | 3 +-- src/libostree/ostree-repo-pull.c | 5 ++--- src/libostree/ostree-repo.c | 5 +---- src/ostree/ot-admin-builtin-instutil.c | 9 ++------- src/ostree/ot-admin-builtin-status.c | 4 +--- src/ostree/ot-builtin-admin.c | 9 ++------- src/ostree/ot-builtin-commit.c | 4 +--- src/ostree/ot-builtin-ls.c | 6 +----- src/ostree/ot-builtin-pull.c | 4 +--- src/ostree/ot-builtin-remote.c | 9 ++------- src/ostree/ot-main.c | 19 +++++-------------- tests/test-gpg-verify-result.c | 3 +-- 13 files changed, 21 insertions(+), 63 deletions(-) diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 2cd02287..63e4b968 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -148,7 +148,7 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot GError **error) { gboolean ret = FALSE; - GString *output = g_string_new (""); + g_autoptr(GString) output = g_string_new (""); g_autoptr(GOutputStream) out_stream = NULL; g_autoptr(GPtrArray) loader_configs = NULL; guint i; @@ -260,8 +260,6 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot ret = TRUE; out: - if (output) - g_string_free (output, TRUE); return ret; } diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 45577373..c42f5971 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -433,7 +433,7 @@ add_size_index_to_metadata (OstreeRepo *self, { guint8 csum[OSTREE_SHA256_DIGEST_LEN]; const char *e_checksum = sorted_keys->pdata[i]; - GString *buffer = g_string_new (NULL); + g_autoptr(GString) buffer = g_string_new (NULL); ostree_checksum_inplace_to_bytes (e_checksum, csum); g_string_append_len (buffer, (char*)csum, sizeof (csum)); @@ -444,7 +444,6 @@ add_size_index_to_metadata (OstreeRepo *self, g_variant_builder_add (&index_builder, "@ay", ot_gvariant_new_bytearray ((guint8*)buffer->str, buffer->len)); - g_string_free (buffer, TRUE); } g_variant_builder_add (builder, "{sv}", "ostree.sizes", diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 12f93ab0..87f57e18 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3474,8 +3474,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, bytes_transferred = _ostree_fetcher_bytes_transferred (pull_data->fetcher); if (bytes_transferred > 0 && pull_data->progress) { - guint shift; - GString *buf = g_string_new (""); + guint shift; + g_autoptr(GString) buf = g_string_new (""); /* Ensure the rest of the progress keys are set appropriately. */ update_progress (pull_data); @@ -3499,7 +3499,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC)); ostree_async_progress_set_status (pull_data->progress, buf->str); - g_string_free (buf, TRUE); } /* iterate over commits fetched and delete any commitpartial files */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1104b91a..e204fd15 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3717,7 +3717,6 @@ void ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data) { - GString *buf; g_autofree char *status = NULL; gboolean scanning; guint outstanding_fetches; @@ -3729,7 +3728,7 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress guint fetched_delta_part_fallbacks; guint total_delta_part_fallbacks; - buf = g_string_new (""); + g_autoptr(GString) buf = g_string_new (""); ostree_async_progress_get (progress, "outstanding-fetches", "u", &outstanding_fetches, @@ -3838,8 +3837,6 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress } glnx_console_text (buf->str); - - g_string_free (buf, TRUE); } /** diff --git a/src/ostree/ot-admin-builtin-instutil.c b/src/ostree/ot-admin-builtin-instutil.c index f27316a3..0694ba77 100644 --- a/src/ostree/ot-admin-builtin-instutil.c +++ b/src/ostree/ot-admin-builtin-instutil.c @@ -47,12 +47,9 @@ static GOptionContext * ostree_admin_instutil_option_context_new_with_commands (void) { OstreeAdminInstUtilCommand *command = admin_instutil_subcommands; - GOptionContext *context; - GString *summary; + GOptionContext *context = g_option_context_new ("COMMAND"); - context = g_option_context_new ("COMMAND"); - - summary = g_string_new ("Builtin \"admin instutil\" Commands:"); + g_autoptr(GString) summary = g_string_new ("Builtin \"admin instutil\" Commands:"); while (command->name != NULL) { @@ -62,8 +59,6 @@ ostree_admin_instutil_option_context_new_with_commands (void) g_option_context_set_summary (context, summary->str); - g_string_free (summary, TRUE); - return context; } diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 940e5df0..79621a1d 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -124,7 +124,6 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro OstreeDeploymentUnlockedState unlocked = ostree_deployment_get_unlocked (deployment); g_autofree char *version = version_of_commit (repo, ref); glnx_unref_object OstreeGpgVerifyResult *result = NULL; - GString *output_buffer; guint jj, n_signatures; GError *local_error = NULL; @@ -159,6 +158,7 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro if (deployment_get_gpg_verify (deployment, repo)) { + g_autoptr(GString) output_buffer = g_string_sized_new (256); /* Print any digital signatures on this commit. */ result = ostree_repo_verify_commit_ext (repo, ref, NULL, NULL, @@ -176,7 +176,6 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro goto out; } - output_buffer = g_string_sized_new (256); n_signatures = ostree_gpg_verify_result_count_all (result); for (jj = 0; jj < n_signatures; jj++) @@ -186,7 +185,6 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro } g_print ("%s", output_buffer->str); - g_string_free (output_buffer, TRUE); } } } diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index 0d8290a8..16aaec03 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -56,12 +56,9 @@ static GOptionContext * ostree_admin_option_context_new_with_commands (void) { OstreeAdminCommand *command = admin_subcommands; - GOptionContext *context; - GString *summary; + GOptionContext *context = g_option_context_new ("--print-current-dir|COMMAND"); - context = g_option_context_new ("--print-current-dir|COMMAND"); - - summary = g_string_new ("Builtin \"admin\" Commands:"); + g_autoptr(GString) summary = g_string_new ("Builtin \"admin\" Commands:"); while (command->name != NULL) { @@ -71,8 +68,6 @@ ostree_admin_option_context_new_with_commands (void) g_option_context_set_summary (context, summary->str); - g_string_free (summary, TRUE); - return context; } diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 53dab6d4..4b0d8821 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -218,7 +218,7 @@ commit_editor (OstreeRepo *repo, g_autofree char *input = NULL; g_autofree char *output = NULL; gboolean ret = FALSE; - GString *bodybuf = NULL; + g_autoptr(GString) bodybuf = NULL; char **lines = NULL; int i; @@ -288,8 +288,6 @@ commit_editor (OstreeRepo *repo, out: g_strfreev (lines); - if (bodybuf) - g_string_free (bodybuf, TRUE); return ret; } diff --git a/src/ostree/ot-builtin-ls.c b/src/ostree/ot-builtin-ls.c index 3abeb5c4..3e0336f6 100644 --- a/src/ostree/ot-builtin-ls.c +++ b/src/ostree/ot-builtin-ls.c @@ -47,15 +47,13 @@ static void print_one_file_text (GFile *f, GFileInfo *file_info) { - GString *buf = NULL; + g_autoptr(GString) buf = g_string_new (""); char type_c; guint32 mode; guint32 type; if (!ostree_repo_file_ensure_resolved ((OstreeRepoFile*)f, NULL)) g_assert_not_reached (); - - buf = g_string_new (""); type_c = '?'; mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); @@ -119,8 +117,6 @@ print_one_file_text (GFile *f, g_string_append_printf (buf, " -> %s", g_file_info_get_attribute_byte_string (file_info, "standard::symlink-target")); g_print ("%s\n", buf->str); - - g_string_free (buf, TRUE); } static void diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 08f0028a..d88c5ee8 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -84,7 +84,6 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, guint fetched_delta_parts, total_delta_parts; guint fetched_delta_part_fallbacks, total_delta_part_fallbacks; guint64 fetched_delta_part_size, total_delta_part_size, total_delta_part_usize; - GString *buf; g_assert (!printed_console_progress); printed_console_progress = TRUE; @@ -107,7 +106,7 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, fetched_delta_parts += fetched_delta_part_fallbacks; total_delta_parts += total_delta_part_fallbacks; - buf = g_string_new (""); + g_autoptr(GString) buf = g_string_new (""); { g_autofree char *formatted_fetched = g_format_size (fetched_delta_part_size); @@ -122,7 +121,6 @@ dry_run_console_progress_changed (OstreeAsyncProgress *progress, formatted_usize); } g_print ("%s\n", buf->str); - g_string_free (buf, TRUE); } gboolean diff --git a/src/ostree/ot-builtin-remote.c b/src/ostree/ot-builtin-remote.c index f0667a42..dd999c01 100644 --- a/src/ostree/ot-builtin-remote.c +++ b/src/ostree/ot-builtin-remote.c @@ -51,12 +51,9 @@ static GOptionContext * remote_option_context_new_with_commands (void) { OstreeRemoteCommand *subcommand = remote_subcommands; - GOptionContext *context; - GString *summary; + GOptionContext *context = g_option_context_new ("COMMAND"); - context = g_option_context_new ("COMMAND"); - - summary = g_string_new ("Builtin \"remote\" Commands:"); + g_autoptr(GString) summary = g_string_new ("Builtin \"remote\" Commands:"); while (subcommand->name != NULL) { @@ -66,8 +63,6 @@ remote_option_context_new_with_commands (void) g_option_context_set_summary (context, summary->str); - g_string_free (summary, TRUE); - return context; } diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index efc240a7..a5b630a0 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -59,12 +59,9 @@ static GOptionEntry global_admin_entries[] = { static GOptionContext * ostree_option_context_new_with_commands (OstreeCommand *commands) { - GOptionContext *context; - GString *summary; + GOptionContext *context = g_option_context_new ("COMMAND"); - context = g_option_context_new ("COMMAND"); - - summary = g_string_new ("Builtin Commands:"); + g_autoptr(GString) summary = g_string_new ("Builtin Commands:"); while (commands->name != NULL) { @@ -74,8 +71,6 @@ ostree_option_context_new_with_commands (OstreeCommand *commands) g_option_context_set_summary (context, summary->str); - g_string_free (summary, TRUE); - return context; } @@ -389,18 +384,15 @@ ostree_ensure_repo_writable (OstreeRepo *repo, void ostree_print_gpg_verify_result (OstreeGpgVerifyResult *result) { - GString *buffer; - guint n_sigs, ii; - - n_sigs = ostree_gpg_verify_result_count_all (result); + guint n_sigs = ostree_gpg_verify_result_count_all (result); /* XXX If we ever add internationalization, use ngettext() here. */ g_print ("GPG: Verification enabled, found %u signature%s:\n", n_sigs, n_sigs == 1 ? "" : "s"); - buffer = g_string_sized_new (256); + g_autoptr(GString) buffer = g_string_sized_new (256); - for (ii = 0; ii < n_sigs; ii++) + for (guint ii = 0; ii < n_sigs; ii++) { g_string_append_c (buffer, '\n'); ostree_gpg_verify_result_describe (result, ii, buffer, " ", @@ -408,7 +400,6 @@ ostree_print_gpg_verify_result (OstreeGpgVerifyResult *result) } g_print ("%s", buffer->str); - g_string_free (buffer, TRUE); } gboolean diff --git a/tests/test-gpg-verify-result.c b/tests/test-gpg-verify-result.c index d2c1ff66..62b05e33 100644 --- a/tests/test-gpg-verify-result.c +++ b/tests/test-gpg-verify-result.c @@ -28,11 +28,10 @@ #define assert_no_gpg_error(err, filename) \ G_STMT_START { \ if (err != GPG_ERR_NO_ERROR) { \ - GString *string = g_string_new ("assertion failed "); \ + g_autoptr(GString) string = g_string_new ("assertion failed "); \ g_string_append_printf (string, "%s: %s ", gpgme_strsource (err), gpgme_strerror (err)); \ g_string_append (string, filename ? filename : ""); \ g_assertion_message (G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC, string->str); \ - g_string_free (string, TRUE); \ } \ } G_STMT_END From e6666fc2e5060ad2a219796d929017d3208da90b Mon Sep 17 00:00:00 2001 From: Sjoerd Simons Date: Tue, 2 May 2017 21:23:08 +0200 Subject: [PATCH 45/94] repo/commit: Fix memory leak While running the testsuite under valgrind a small memory leak showed up: ==16487== 65 bytes in 1 blocks are definitely lost in loss record 773 of 1,123 ==16487== at 0x4C2BBAF: malloc (vg_replace_malloc.c:299) ==16487== by 0x6048E08: g_malloc (gmem.c:94) ==16487== by 0x6062EAE: g_strdup (gstrfuncs.c:363) ==16487== by 0x54CE3E6: write_object (ostree-repo-commit.c:776) ==16487== by 0x54CF2D4: ostree_repo_write_metadata (ostree-repo-commit.c:1528) ==16487== by 0x54CF505: _ostree_repo_write_directory_meta (ostree-repo-commit.c:1712) ==16487== by 0x54D0AB4: write_dfd_iter_to_mtree_internal (ostree-repo-commit.c:2650) ==16487== by 0x54D0E2D: ostree_repo_write_dfd_to_mtree (ostree-repo-commit.c:2793) ==16487== by 0x1190C4: ostree_builtin_commit (ot-builtin-commit.c:474) ==16487== by 0x11F2EE: ostree_run (ot-main.c:200) ==16487== by 0x116F32: main (main.c:78) The reason for this is that ot_checksum_instream_get_string returns a chunk of newly allocated memory which never got freed. Make actual_checksum something that gets autocleanend and own the memory assigned to it in all cases. Signed-off-by: Sjoerd Simons Closes: #827 Approved by: pwithnall --- src/libostree/ostree-repo-commit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index c42f5971..487bd370 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -607,7 +607,8 @@ write_object (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - const char *actual_checksum; + const char *actual_checksum = NULL; + g_autofree char *actual_checksum_owned = NULL; gboolean do_commit; OstreeRepoMode repo_mode; g_autofree char *temp_filename = NULL; @@ -772,7 +773,7 @@ write_object (OstreeRepo *self, actual_checksum = expected_checksum; else { - actual_checksum = ot_checksum_instream_get_string (checksum_input); + actual_checksum = actual_checksum_owned = ot_checksum_instream_get_string (checksum_input); if (expected_checksum && strcmp (actual_checksum, expected_checksum) != 0) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, From 9aa8d420cf92335a1de08040f96f2b75dd99ccb5 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Tue, 2 May 2017 22:22:15 +0100 Subject: [PATCH 46/94] libostree: Add some additional metadata to the summary file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Commit timestamps, so it’s easy to work out whether a given commit is newer than the one we have locally • Summary file timestamp, so it’s easy to work out whether the summary file is more up to date than another summary file • Summary file expiry time, so clients can work out when they should expect the summary file to next be updated, and hence can query for it at roughly the right time The expiry time requires input from the user, so is currently never set automatically. Programs using libostree can set it if they wish. Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- src/libostree/ostree-core.h | 11 +++++++++++ src/libostree/ostree-repo-private.h | 8 ++++++++ src/libostree/ostree-repo.c | 26 ++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index b25112db..59d6d9e2 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -158,6 +158,17 @@ typedef enum { * - a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional metadata), sorted by ref name * - a{sv} - Additional metadata, at the current time the following are defined: * - key: "ostree.static-deltas", value: a{sv}, static delta name -> 32 bytes of checksum + * - key: "ostree.summary.last-modified", value: t, timestamp (seconds since + * the Unix epoch in UTC, big-endian) when the summary was last regenerated + * (similar to the HTTP `Last-Modified` header) + * - key: "ostree.summary.expires", value: t, timestamp (seconds since the + * Unix epoch in UTC, big-endian) after which the summary is considered + * stale and should be re-downloaded if possible (similar to the HTTP + * `Expires` header) + * + * The currently defined keys for the `a{sv}` of additional metadata for each commit are: + * - key: `ostree.commit.timestamp`, value: `t`, timestamp (seconds since the + * Unix epoch in UTC, big-endian) when the commit was committed */ #define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})" #define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 94330226..87e67a23 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -43,6 +43,14 @@ G_BEGIN_DECLS * */ #define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16 +/* Well-known keys for the additional metadata field in a summary file. */ +#define OSTREE_SUMMARY_LAST_MODIFIED "ostree.summary.last-modified" +#define OSTREE_SUMMARY_EXPIRES "ostree.summary.expires" + +/* Well-known keys for the additional metadata field in a commit in a ref entry + * in a summary file. */ +#define OSTREE_COMMIT_TIMESTAMP "ostree.commit.timestamp" + typedef enum { OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0) } OstreeRepoTestErrorFlags; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index e204fd15..fd7aa55d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4563,6 +4563,10 @@ ostree_repo_verify_summary (OstreeRepo *self, * An OSTree repository can contain a high level "summary" file that * describes the available branches and other metadata. * + * If the timetable for making commits and updating the summary file is fairly + * regular, setting the `ostree.summary.expires` key in @additional_metadata + * will aid clients in working out when to check for updates. + * * It is regenerated automatically after a commit if * `core/commit-update-summary` is set. */ @@ -4595,6 +4599,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, const char *commit = g_hash_table_lookup (refs, ref); g_autofree char *remotename = NULL; g_autoptr(GVariant) commit_obj = NULL; + g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; + guint64 commit_timestamp; + g_autoptr(GDateTime) dt = NULL; g_assert (commit); @@ -4608,11 +4615,21 @@ ostree_repo_regenerate_summary (OstreeRepo *self, if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) goto out; - g_variant_builder_add_value (refs_builder, + g_variant_dict_init (&commit_metadata_builder, NULL); + + /* Forward the commit’s timestamp if it’s valid. */ + commit_timestamp = ostree_commit_get_timestamp (commit_obj); + dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt != NULL) + g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, + g_variant_new_uint64 (GUINT64_TO_BE (commit_timestamp))); + + g_variant_builder_add_value (refs_builder, g_variant_new ("(s(t@ay@a{sv}))", ref, (guint64) g_variant_get_size (commit_obj), ostree_checksum_to_bytes_v (commit), - ot_gvariant_new_empty_string_dict ())); + g_variant_dict_end (&commit_metadata_builder))); } @@ -4661,6 +4678,11 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); } + { + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED, + g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); + } + { g_autoptr(GVariantBuilder) summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT); From 015ce7520b3099a0af110f7ee693ce40019c2445 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 4 May 2017 11:01:41 +0100 Subject: [PATCH 47/94] libostree: Document endianness of GVariant metadata types Endianness strikes again. Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- src/libostree/ostree-core.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 59d6d9e2..c530cea3 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -104,9 +104,9 @@ typedef enum { /** * OSTREE_DIRMETA_GVARIANT_FORMAT: * - * - u - uid - * - u - gid - * - u - mode + * - u - uid (big-endian) + * - u - gid (big-endian) + * - u - mode (big-endian) * - a(ayay) - xattrs */ #define OSTREE_DIRMETA_GVARIANT_STRING "(uuua(ayay))" @@ -120,9 +120,9 @@ typedef enum { * can't store in the real filesystem but we can still use a regular .file object * that we can hardlink to in the case of a user-mode checkout. * - * - u - uid - * - u - gid - * - u - mode + * - u - uid (big-endian) + * - u - gid (big-endian) + * - u - mode (big-endian) * - a(ayay) - xattrs */ #define OSTREE_FILEMETA_GVARIANT_STRING "(uuua(ayay))" @@ -145,7 +145,7 @@ typedef enum { * - a(say) - Related objects * - s - subject * - s - body - * - t - Timestamp in seconds since the epoch (UTC) + * - t - Timestamp in seconds since the epoch (UTC, big-endian) * - ay - Root tree contents * - ay - Root tree metadata */ From 8ea654251a61d080e8c8553c9905e02599059294 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 4 May 2017 11:31:50 +0100 Subject: [PATCH 48/94] ostree: Add --view mode to `ostree summary` This allows a locally generated summary file to be viewed. It accepts the same arguments as `ostree remote summary` (i.e. --raw). Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- src/ostree/ot-builtin-summary.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index fa037400..9055d972 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -20,17 +20,21 @@ #include "config.h" +#include "ostree-repo-private.h" +#include "ot-dump.h" #include "ot-main.h" #include "ot-builtins.h" #include "ostree.h" #include "otutil.h" -static gboolean opt_update; +static gboolean opt_update, opt_view, opt_raw; static char **opt_key_ids; static char *opt_gpg_homedir; static GOptionEntry options[] = { { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, + { "view", 'v', 0, G_OPTION_ARG_NONE, &opt_view, "View the local summary file", NULL }, + { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "View the raw bytes of the summary file", NULL }, { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, { NULL } @@ -42,6 +46,7 @@ ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError gboolean ret = FALSE; g_autoptr(GOptionContext) context = NULL; glnx_unref_object OstreeRepo *repo = NULL; + OstreeDumpFlags flags = OSTREE_DUMP_NONE; context = g_option_context_new ("Manage summary metadata"); @@ -66,6 +71,19 @@ ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError goto out; } } + else if (opt_view) + { + g_autoptr(GBytes) summary_data = NULL; + + if (opt_raw) + flags |= OSTREE_DUMP_RAW; + + summary_data = ot_file_mapat_bytes (repo->repo_dir_fd, "summary", error); + if (!summary_data) + goto out; + + ot_dump_summary_bytes (summary_data, flags); + } else { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, From a946c3d42385d7ab44a01dcf212400cd5df28be3 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 4 May 2017 11:32:46 +0100 Subject: [PATCH 49/94] ostree: Improve formatting for well-known summary metadata keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a summary metadata key is well-known, like ostree.static-deltas, or ostree.summary.last-modified, format it a little more nicely. This is especially important for timestamps like last-modified, since otherwise they’re formatted as a big-endian uint64, which is basically unusable for the user. Non-formatted output can still be retrieved using the OSTREE_DUMP_RAW flag, and the non-formatted key name is always printed for clarity. Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- src/ostree/ot-dump.c | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 4fc84cbd..7cbc2945 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -199,6 +199,18 @@ dump_summary_ref (const char *ref_name, } } +static gchar * +uint64_secs_to_iso8601 (guint64 secs) +{ + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); + g_autoptr(GDateTime) local = (dt != NULL) ? g_date_time_to_local (dt) : NULL; + + if (local != NULL) + return g_date_time_format (local, "%FT%T%:::z"); + else + return g_strdup ("invalid"); +} + void ot_dump_summary_bytes (GBytes *summary_bytes, OstreeDumpFlags flags) @@ -251,11 +263,35 @@ ot_dump_summary_bytes (GBytes *summary_bytes, g_variant_iter_init (&iter, exts); - /* XXX Should we print something more human-friendly for - * known extension names like 'ostree.static-deltas'? */ while (g_variant_iter_loop (&iter, "{sv}", &key, &value)) { - g_autofree char *string = g_variant_print (value, FALSE); - g_print ("%s: %s\n", key, string); + g_autofree gchar *value_str = NULL; + const gchar *pretty_key = NULL; + + if (g_strcmp0 (key, "ostree.static-deltas") == 0) + { + pretty_key = "Static Deltas"; + value_str = g_variant_print (value, FALSE); + } + else if (g_strcmp0 (key, "ostree.summary.last-modified") == 0) + { + pretty_key = "Last-Modified"; + value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); + } + else if (g_strcmp0 (key, "ostree.summary.expires") == 0) + { + pretty_key = "Expires"; + value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); + } + else + { + value_str = g_variant_print (value, FALSE); + } + + /* Print out. */ + if (pretty_key != NULL) + g_print ("%s (%s): %s\n", pretty_key, key, value_str); + else + g_print ("%s: %s\n", key, value_str); } } From c1290177a312d5f416d7afd43be2c1cfe1a3f7ad Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 5 May 2017 11:28:42 +0100 Subject: [PATCH 50/94] ostree: Use #defines for well-known metadata key names Rather than hard-coding the names as strings. This makes the code a little more maintainable. Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- src/ostree/ot-builtin-gpg-sign.c | 7 ++++--- src/ostree/ot-dump.c | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-gpg-sign.c b/src/ostree/ot-builtin-gpg-sign.c index 0ef2be8b..e14eba68 100644 --- a/src/ostree/ot-builtin-gpg-sign.c +++ b/src/ostree/ot-builtin-gpg-sign.c @@ -26,6 +26,7 @@ #include "ot-builtins.h" #include "ostree.h" #include "otutil.h" +#include "ostree-core-private.h" static gboolean opt_delete; static char *opt_gpg_homedir; @@ -85,7 +86,7 @@ delete_signatures (OstreeRepo *repo, g_variant_dict_init (&metadata_dict, old_metadata); signature_data = g_variant_dict_lookup_value (&metadata_dict, - "ostree.gpgsigs", + _OSTREE_METADATA_GPGSIGS_NAME, G_VARIANT_TYPE ("aay")); /* Taking the approach of deleting whatever matches we find for the @@ -154,7 +155,7 @@ delete_signatures (OstreeRepo *repo, /* Update the metadata dictionary. */ if (g_queue_is_empty (&signatures)) { - g_variant_dict_remove (&metadata_dict, "ostree.gpgsigs"); + g_variant_dict_remove (&metadata_dict, _OSTREE_METADATA_GPGSIGS_NAME); } else { @@ -170,7 +171,7 @@ delete_signatures (OstreeRepo *repo, } g_variant_dict_insert_value (&metadata_dict, - "ostree.gpgsigs", + _OSTREE_METADATA_GPGSIGS_NAME, g_variant_builder_end (&signature_builder)); } diff --git a/src/ostree/ot-dump.c b/src/ostree/ot-dump.c index 7cbc2945..b24003c7 100644 --- a/src/ostree/ot-dump.c +++ b/src/ostree/ot-dump.c @@ -26,6 +26,8 @@ #include +#include "ostree-repo-private.h" +#include "ostree-repo-static-delta-private.h" #include "ot-dump.h" #include "otutil.h" #include "ot-admin-functions.h" @@ -268,17 +270,17 @@ ot_dump_summary_bytes (GBytes *summary_bytes, g_autofree gchar *value_str = NULL; const gchar *pretty_key = NULL; - if (g_strcmp0 (key, "ostree.static-deltas") == 0) + if (g_strcmp0 (key, OSTREE_SUMMARY_STATIC_DELTAS) == 0) { pretty_key = "Static Deltas"; value_str = g_variant_print (value, FALSE); } - else if (g_strcmp0 (key, "ostree.summary.last-modified") == 0) + else if (g_strcmp0 (key, OSTREE_SUMMARY_LAST_MODIFIED) == 0) { pretty_key = "Last-Modified"; value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); } - else if (g_strcmp0 (key, "ostree.summary.expires") == 0) + else if (g_strcmp0 (key, OSTREE_SUMMARY_EXPIRES) == 0) { pretty_key = "Expires"; value_str = uint64_secs_to_iso8601 (GUINT64_FROM_BE (g_variant_get_uint64 (value))); From f3cc0eb25a773d21fdbbb778f1756df5466ff7cf Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Sun, 7 May 2017 18:55:29 +0100 Subject: [PATCH 51/94] tests: Add a test for `ostree summary --view` This includes a test of the new human-readable key names. Signed-off-by: Philip Withnall Closes: #826 Approved by: cgwalters --- Makefile-tests.am | 1 + tests/test-summary-view.sh | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 tests/test-summary-view.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index a0eefdb9..2c3e5047 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -101,6 +101,7 @@ dist_test_scripts = \ tests/test-switchroot.sh \ tests/test-pull-contenturl.sh \ tests/test-pull-mirrorlist.sh \ + tests/test-summary-view.sh \ tests/coccinelle.sh \ $(NULL) diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh new file mode 100755 index 00000000..afaa5856 --- /dev/null +++ b/tests/test-summary-view.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Copyright © 2017 Endless Mobile, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# +# Authors: +# - Philip Withnall + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..2" + +COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}" +setup_fake_remote_repo1 "archive-z2" "${COMMIT_SIGN}" + +# Set up a second branch. +mkdir ${test_tmpdir}/ostree-srv/other-files +cd ${test_tmpdir}/ostree-srv/other-files +echo 'hello world some object' > hello-world +${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_SIGN} -b other -s "A commit" -m "Example commit body" + +# Generate the summary file. +${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u + +# Check out the repository. +prev_dir=`pwd` +cd ${test_tmpdir} +ostree_repo_init repo --mode=archive-z2 +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo +${CMD_PREFIX} ostree --repo=repo pull --mirror origin + +# Check the summary file exists in the checkout, and can be viewed. +assert_has_file repo/summary +output=$(${OSTREE} summary --view) +echo "$output" | sed -e 's/^/# /' +echo "$output" | grep --quiet --no-messages "* main" +echo "$output" | grep --quiet --no-messages "* other" +echo "$output" | grep --quiet --no-messages "ostree.summary.last-modified" +echo "$output" | grep --quiet --no-messages "Static Deltas (ostree.static-deltas): {}" +echo "ok view summary" + +# Check the summary can be viewed raw too. +raw_output=$(${OSTREE} summary --view --raw) +echo "$raw_output" | sed -e 's/^/# /' +echo "$raw_output" | grep --quiet --no-messages "('main', (" +echo "$raw_output" | grep --quiet --no-messages "('other', (" +echo "$raw_output" | grep --quiet --no-messages "{'ostree.summary.last-modified': Date: Thu, 4 May 2017 14:10:57 -0500 Subject: [PATCH 52/94] commit: Mark ostree_repo_transaction_set_ref* checksums nullable Allow GI bindings to delete refs through ostree_repo_transaction_set_ref and ostree_repo_transaction_set_refspec by setting the checksum to NULL. Closes: #834 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 4 ++-- tests/test-core.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 487bd370..664016b8 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1306,7 +1306,7 @@ ensure_txn_refs (OstreeRepo *self) * ostree_repo_transaction_set_refspec: * @self: An #OstreeRepo * @refspec: The refspec to write - * @checksum: The checksum to point it to + * @checksum: (nullable): The checksum to point it to * * Like ostree_repo_transaction_set_ref(), but takes concatenated * @refspec format as input instead of separate remote and name @@ -1329,7 +1329,7 @@ ostree_repo_transaction_set_refspec (OstreeRepo *self, * @self: An #OstreeRepo * @remote: (allow-none): A remote for the ref * @ref: The ref to write - * @checksum: The checksum to point it to + * @checksum: (nullable): The checksum to point it to * * If @checksum is not %NULL, then record it as the target of ref named * @ref; if @remote is provided, the ref will appear to originate from that diff --git a/tests/test-core.js b/tests/test-core.js index e9ace6e9..64d1b62d 100644 --- a/tests/test-core.js +++ b/tests/test-core.js @@ -52,4 +52,18 @@ let child = root.get_child('some-file'); let info = child.query_info("standard::name,standard::type,standard::size", 0, null); assertEquals(info.get_size(), 12); +// Write a ref and read it back +repo.prepare_transaction(null); +repo.transaction_set_refspec('someref', commit); +repo.commit_transaction(null, null); +let [,readCommit] = repo.resolve_rev('someref', false); +assertEquals(readCommit, commit); + +// Delete a ref +repo.prepare_transaction(null); +repo.transaction_set_refspec('someref', null); +repo.commit_transaction(null, null); +[,readCommit] = repo.resolve_rev('someref', true); +assertEquals(readCommit, null); + print("test-core complete"); From 4e13361c8e471822613cfad65e5b39f8ec6b91b7 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 8 May 2017 09:46:35 -0500 Subject: [PATCH 53/94] pull: Allow additional HTTP headers for summary fetch Read the http-headers (a(ss)) option in ostree_repo_remote_fetch_summary_with_options like ostree_repo_pull_with_options and add the headers to the fetcher. This allows things like providing additional authorization headers to the HTTP requests. Closes: #839 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 87f57e18..7929b91a 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2460,10 +2460,14 @@ repo_remote_fetch_summary (OstreeRepo *self, gboolean ret = FALSE; gboolean from_cache = FALSE; const char *url_override = NULL; + g_autoptr(GVariant) extra_headers = NULL; g_autoptr(GPtrArray) mirrorlist = NULL; if (options) - (void) g_variant_lookup (options, "override-url", "&s", &url_override); + { + (void) g_variant_lookup (options, "override-url", "&s", &url_override); + (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); + } mainctx = g_main_context_new (); g_main_context_push_thread_default (mainctx); @@ -2472,6 +2476,9 @@ repo_remote_fetch_summary (OstreeRepo *self, if (fetcher == NULL) goto out; + if (extra_headers) + _ostree_fetcher_set_extra_headers (fetcher, extra_headers); + { g_autofree char *url_string = NULL; if (metalink_url_string) @@ -3584,6 +3591,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, * The following are currently defined: * * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options + * - http-headers (a(ss)): Additional headers to add to all HTTP requests * * Returns: %TRUE on success, %FALSE on failure */ From 9690a54e47276e91fb022822498cb287c35d6fdb Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 8 May 2017 16:04:26 +0100 Subject: [PATCH 54/94] tests: Fix regex escaping in test-summary-view.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There were some regex special characters in the pattern strings, which I think were causing the test to fail on some Travis builds due to using an invalid regex. Fix that by matching using fixed strings instead. We don’t need regexes here. Use a new assert_file_has_content_literal to do that for us. Signed-off-by: Philip Withnall Closes: #838 Approved by: cgwalters --- tests/libtest-core.sh | 7 +++++++ tests/test-summary-view.sh | 20 +++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index ae7f381f..d1d3bbdf 100644 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -85,6 +85,13 @@ assert_file_has_content () { fi } +assert_file_has_content_literal () { + if ! grep -q -F -e "$2" "$1"; then + sed -e 's/^/# /' < "$1" >&2 + fatal "File '$1' doesn't match fixed string list '$2'" + fi +} + assert_symlink_has_content () { if ! test -L "$1"; then echo 1>&2 "File '$1' is not a symbolic link" diff --git a/tests/test-summary-view.sh b/tests/test-summary-view.sh index afaa5856..6dcfd088 100755 --- a/tests/test-summary-view.sh +++ b/tests/test-summary-view.sh @@ -47,20 +47,18 @@ ${CMD_PREFIX} ostree --repo=repo pull --mirror origin # Check the summary file exists in the checkout, and can be viewed. assert_has_file repo/summary -output=$(${OSTREE} summary --view) -echo "$output" | sed -e 's/^/# /' -echo "$output" | grep --quiet --no-messages "* main" -echo "$output" | grep --quiet --no-messages "* other" -echo "$output" | grep --quiet --no-messages "ostree.summary.last-modified" -echo "$output" | grep --quiet --no-messages "Static Deltas (ostree.static-deltas): {}" +${OSTREE} summary --view > summary.txt +assert_file_has_content_literal summary.txt "* main" +assert_file_has_content_literal summary.txt "* other" +assert_file_has_content_literal summary.txt "ostree.summary.last-modified" +assert_file_has_content_literal summary.txt "Static Deltas (ostree.static-deltas): {}" echo "ok view summary" # Check the summary can be viewed raw too. -raw_output=$(${OSTREE} summary --view --raw) -echo "$raw_output" | sed -e 's/^/# /' -echo "$raw_output" | grep --quiet --no-messages "('main', (" -echo "$raw_output" | grep --quiet --no-messages "('other', (" -echo "$raw_output" | grep --quiet --no-messages "{'ostree.summary.last-modified': raw-summary.txt +assert_file_has_content_literal raw-summary.txt "('main', (" +assert_file_has_content_literal raw-summary.txt "('other', (" +assert_file_has_content_literal raw-summary.txt "{'ostree.summary.last-modified': Date: Thu, 4 May 2017 14:16:21 -0500 Subject: [PATCH 55/94] tests: Install libtest-core.sh with installed tests Without this, running the installed tests fails dramatically. Remove it from EXTRA_DIST since dist_installed_test_data takes care of dist. Closes: #837 Approved by: dbnicholson --- Makefile-tests.am | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 2c3e5047..6cbce991 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -129,9 +129,10 @@ dist_installed_test_data = tests/archive-test.sh \ tests/basic-test.sh \ tests/pre-endian-deltas-repo-big.tar.xz \ tests/pre-endian-deltas-repo-little.tar.xz \ + tests/libtest-core.sh \ $(NULL) -EXTRA_DIST += tests/libtest.sh tests/libtest-core.sh +EXTRA_DIST += tests/libtest.sh dist_test_extra_scripts = \ tests/bootloader-entries-crosscheck.py \ From fa4e4bf4df0c202f518c313f75287caa513da77d Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 4 May 2017 14:58:07 -0500 Subject: [PATCH 56/94] tests: Look for trivial-httpd in $libexecdir Since b825aac, trivial-httpd is in $libexecdir/libostree by default and not available through the ostree runner in PATH. Try to adjust find it when running the tests installed. Closes: #837 Approved by: dbnicholson --- tests/libtest.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 8127982d..3ce718f9 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -105,7 +105,15 @@ fi if test -n "${OSTREE_UNINSTALLED:-}"; then OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd else - OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd" + # trivial-httpd is now in $libexecdir by default, which we don't + # know at this point. Fortunately, libtest.sh is also in + # $libexecdir, so make an educated guess. If it's not found, assume + # it's still runnable as "ostree trivial-httpd". + if [ -x "${test_srcdir}/../../libostree/ostree-trivial-httpd" ]; then + OSTREE_HTTPD="${CMD_PREFIX} ${test_srcdir}/../../libostree/ostree-trivial-httpd" + else + OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd" + fi fi assert_files_hardlinked() { From c7efe01520d27215be8efa16859d3897896473e4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 May 2017 10:44:43 -0400 Subject: [PATCH 57/94] Add --enable-installed-tests=exclusive, fix installed case The major reason to do this is that running tests *both* installed and uninstalled in our CI is a mostly pointless waste of time. Particularly given we have a few expensive tests. We *do* have tests that only run uninstalled (since they require the source code) like `test-symbols.sh`. Hence, add `--enable-installed-tests=exclusive` to mean *only* do installed for most tests. We'll still have uninstalled coverage via the Travis/Debian configs, and we could perhaps do another build with a subset of uninstalled tests, but I'm not really concerned about it. I'd like to do a renewed push for the InstalledTests model since I feel it's just fundamentally better. (`g-d-t-r` kind of sucks, but then so does the automake runner). Also while we're here - fix the CI to use the correct context, which started this mess. Closes: #837 Approved by: dbnicholson --- .redhat-ci.yml | 11 ++++++----- Makefile-tests.am | 35 ++++++++++++++++++++++++++--------- buildutil/glibtests.m4 | 8 ++++++-- configure.ac | 1 + tests/ci-build.sh | 7 ------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 8b58f80c..818a072a 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -23,14 +23,16 @@ build: config-opts: > --prefix=/usr --libdir=/usr/lib64 - --enable-installed-tests + --enable-installed-tests=exclusive --enable-gtk-doc +# The g-d-t-r timeout is for test-pull-many.sh; if tweaking this, +# also be sure to change the other cases below tests: - make syntax-check - ./tests/ci-commitmessage-submodules.sh - make check - - gnome-desktop-testing-runner -p 0 ostree + - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' timeout: 30m @@ -65,7 +67,6 @@ build: config-opts: > --prefix=/usr --libdir=/usr/lib64 - --enable-installed-tests --enable-gtk-doc --enable-rust @@ -90,14 +91,14 @@ build: config-opts: > --prefix=/usr --libdir=/usr/lib64 - --enable-installed-tests + --enable-installed-tests=exclusive --enable-gtk-doc --with-curl --with-openssl tests: - make check - - gnome-desktop-testing-runner -p 0 ostree + - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/' artifacts: - test-suite.log diff --git a/Makefile-tests.am b/Makefile-tests.am index 6cbce991..ab7750d2 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -42,9 +42,16 @@ endif uninstalled_test_data = tests/ostree-symlink-stamp tests/ostree-prepare-root-symlink-stamp \ tests/ostree-remount-symlink-stamp tests/rofiles-fuse-symlink-stamp -dist_uninstalled_test_scripts = tests/test-symbols.sh +dist_uninstalled_test_scripts = tests/test-symbols.sh tests/coccinelle.sh -dist_test_scripts = \ +# This logic implements ENABLE_INSTALLED_TESTS_EXCLUSIVE; see below. +# The goal here if installed tests are enabled, we explicitly make the +# tests *only* run installed, to avoid having to run them twice in CI. +# This overrides the glib-tap.mk emphasis on doing both, if we'd +# used e.g. `dist_test_scripts`. +dist_test_scripts = $(NULL) +test_programs = $(NULL) +_installed_or_uninstalled_test_scripts = \ tests/test-basic.sh \ tests/test-basic-user.sh \ tests/test-basic-user-only.sh \ @@ -102,23 +109,22 @@ dist_test_scripts = \ tests/test-pull-contenturl.sh \ tests/test-pull-mirrorlist.sh \ tests/test-summary-view.sh \ - tests/coccinelle.sh \ $(NULL) if BUILDOPT_FUSE -dist_test_scripts += tests/test-rofiles-fuse.sh +_installed_or_uninstalled_test_scripts += tests/test-rofiles-fuse.sh else EXTRA_DIST += tests/test-rofiles-fuse.sh endif if USE_LIBSOUP -dist_test_scripts += tests/test-remote-cookies.sh +_installed_or_uninstalled_test_scripts += tests/test-remote-cookies.sh endif -# These call into gjs scripts +# These call into gjs scripts js_tests = tests/test-corruption.sh tests/test-pull-corruption.sh if BUILDOPT_GJS -dist_test_scripts += $(js_tests) +_installed_or_uninstalled_test_scripts += $(js_tests) else EXTRA_DIST += $(js_tests) endif @@ -182,7 +188,7 @@ if !ENABLE_INSTALLED_TESTS libreaddir_rand_la_LDFLAGS += -rpath $(abs_builddir) endif -test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ +_installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ tests/test-basic-c tests/test-sysroot-c tests/test-pull-c @@ -191,7 +197,7 @@ test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tes noinst_PROGRAMS += tests/test-rollsum-cli if USE_LIBARCHIVE -test_programs += tests/test-libarchive-import +_installed_or_uninstalled_test_programs += tests/test-libarchive-import endif common_tests_cflags = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx @@ -288,6 +294,17 @@ tests/%-symlink-stamp: % Makefile ln -sf "$${real_bin}" tests/$*; \ touch $@ +# See above comment on binding the tests to be either installed or not. +if ENABLE_INSTALLED_TESTS_EXCLUSIVE +dist_installed_test_scripts = $(_installed_or_uninstalled_test_scripts) +installed_test_programs = $(_installed_or_uninstalled_test_programs) +check-local: + echo "NOTE: Exclusive installed tests are enabled; to run them, make install, then: gnome-desktop-testing-runner -p 0 libostree/" +else +dist_test_scripts += $(_installed_or_uninstalled_test_scripts) +test_programs += $(_installed_or_uninstalled_test_programs) +endif + # Unfortunately the glib test data APIs don't actually handle # non-recursive Automake, so we change our code to canonically look # for tests/ which is just a symlink when installed. diff --git a/buildutil/glibtests.m4 b/buildutil/glibtests.m4 index 27e90246..108c8478 100644 --- a/buildutil/glibtests.m4 +++ b/buildutil/glibtests.m4 @@ -1,17 +1,21 @@ dnl GLIB_TESTS -dnl +dnl NOTE: this file has been modified from upstream glib; see +dnl https://github.com/ostreedev/ostree/pull/837 AC_DEFUN([GLIB_TESTS], [ AC_ARG_ENABLE(installed-tests, AS_HELP_STRING([--enable-installed-tests], [Enable installation of some test cases]), - [case ${enableval} in + [enable_installed_tests=${enableval}; + case ${enableval} in yes) ENABLE_INSTALLED_TESTS="1" ;; + exclusive) ENABLE_INSTALLED_TESTS="1"; ENABLE_INSTALLED_TESTS_EXCLUSIVE=1 ;; no) ENABLE_INSTALLED_TESTS="" ;; *) AC_MSG_ERROR([bad value ${enableval} for --enable-installed-tests]) ;; esac]) AM_CONDITIONAL([ENABLE_INSTALLED_TESTS], test "$ENABLE_INSTALLED_TESTS" = "1") + AM_CONDITIONAL([ENABLE_INSTALLED_TESTS_EXCLUSIVE], test "$ENABLE_INSTALLED_TESTS_EXCLUSIVE" = "1") AC_ARG_ENABLE(always-build-tests, AS_HELP_STRING([--enable-always-build-tests], [Enable always building tests during 'make all']), diff --git a/configure.ac b/configure.ac index c8e02930..dbcc99ea 100644 --- a/configure.ac +++ b/configure.ac @@ -457,6 +457,7 @@ echo " wrpseudo-compat: $enable_wrpseudo_compat man pages (xsltproc): $enable_man api docs (gtk-doc): $enable_gtk_doc + installed tests: $enable_installed_tests gjs-based tests: $have_gjs dracut: $with_dracut mkinitcpio: $with_mkinitcpio diff --git a/tests/ci-build.sh b/tests/ci-build.sh index 23eacf07..e310ed0f 100755 --- a/tests/ci-build.sh +++ b/tests/ci-build.sh @@ -85,7 +85,6 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ../configure \ --enable-always-build-tests \ - --enable-installed-tests \ ${ci_configopts} "$@" @@ -106,12 +105,6 @@ if [ "$ci_sudo" = yes ] && [ "$ci_test" = yes ]; then ${make} installcheck || \ maybe_fail_tests cat test/test-suite.log || : - - env \ - LD_LIBRARY_PATH=/usr/local/lib \ - GI_TYPELIB_PATH=/usr/local/lib/girepository-1.0 \ - gnome-desktop-testing-runner -d /usr/local/share ostree/ || \ - maybe_fail_tests fi # vim:set sw=4 sts=4 et: From c9244b1bb2152954ecadb27f9285ae51d1055d6d Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 1 May 2017 10:59:33 +0100 Subject: [PATCH 58/94] build: Add --enable-experimental-api configure option for unstable APIs There are currently no unstable APIs, but some will be added in following commits. They will be built and exposed in the libostree global symbol list iff configured with --enable-experimental-api. Distributions should not package OSTree with --enable-experimental-api. This is designed for previewing new APIs on controlled platforms; any of the APIs hidden behind this option may be changed or removed at any point. Signed-off-by: Philip Withnall Closes: #832 Approved by: cgwalters --- Makefile-libostree.am | 10 +++++++- Makefile-tests.am | 1 + apidoc/Makefile.am | 1 + apidoc/ostree-experimental-sections.txt | 0 configure.ac | 19 +++++++++++++- src/libostree/libostree-experimental.sym | 32 ++++++++++++++++++++++++ tests/test-symbols.sh | 14 ++++++++--- 7 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 apidoc/ostree-experimental-sections.txt create mode 100644 src/libostree/libostree-experimental.sym diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 0a114884..8d44b61c 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -156,7 +156,15 @@ libostree_1_la_LIBADD = libotutil.la libglnx.la libbsdiff.la libostree-kernel-ar libostree_1_la_LIBADD += $(bupsplitpath) EXTRA_libostree_1_la_DEPENDENCIES = $(top_srcdir)/src/libostree/libostree.sym -EXTRA_DIST += src/libostree/libostree.sym +EXTRA_DIST += \ + src/libostree/libostree.sym \ + src/libostree/libostree-experimental.sym \ + $(NULL) + +if ENABLE_EXPERIMENTAL_API +libostree_1_la_LDFLAGS += -Wl,--version-script=$(top_srcdir)/src/libostree/libostree-experimental.sym +EXTRA_libostree_1_la_DEPENDENCIES += $(top_srcdir)/src/libostree/libostree-experimental.sym +endif if USE_LIBARCHIVE libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) diff --git a/Makefile-tests.am b/Makefile-tests.am index ab7750d2..14e30b46 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -34,6 +34,7 @@ TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd)$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH} \ LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd)$${LD_LIBRARY_PATH:+:$${LD_LIBRARY_PATH}} \ PATH=$$(cd $(top_builddir)/tests && pwd):$${PATH} \ + OSTREE_FEATURES="$(OSTREE_FEATURES)" \ $(NULL) if BUILDOPT_ASAN TESTS_ENVIRONMENT += OT_SKIP_READDIR_RAND=1 G_SLICE=always-malloc diff --git a/apidoc/Makefile.am b/apidoc/Makefile.am index 730a4c33..f3405fb0 100644 --- a/apidoc/Makefile.am +++ b/apidoc/Makefile.am @@ -120,6 +120,7 @@ include $(top_srcdir)/gtk-doc.make EXTRA_DIST += \ version.xml \ ostree-sections.txt \ + ostree-experimental-sections.txt \ $(NULL) -include $(top_srcdir)/git.mk diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt new file mode 100644 index 00000000..e69de29b diff --git a/configure.ac b/configure.ac index dbcc99ea..6acd0b3c 100644 --- a/configure.ac +++ b/configure.ac @@ -429,6 +429,22 @@ AS_IF([test "x$found_introspection" = xyes && test x$using_asan != xyes], [ ], [have_gjs=no]) AM_CONDITIONAL(BUILDOPT_GJS, test x$have_gjs = xyes) +# Do we enable building experimental (non-stable) API? +# The OSTREE_ENABLE_EXPERIMENTAL_API #define is used internally and in public +# headers, so any consumer of libostree who wants to use experimental API must +# #define OSTREE_ENABLE_EXPERIMENTAL_API 1 +# before including libostree headers. This means the name in the AC_DEFINE below +# is public API. +AC_ARG_ENABLE([experimental-api], + [AS_HELP_STRING([--enable-experimental-api], + [Enable unstable experimental API in libostree [default=no]])],, + [enable_experimental_api=no]) +AS_IF([test x$enable_experimental_api = xyes], + [AC_DEFINE([OSTREE_ENABLE_EXPERIMENTAL_API],[1],[Define if experimental API should be enabled]) + OSTREE_FEATURES="$OSTREE_FEATURES experimental"] +) +AM_CONDITIONAL([ENABLE_EXPERIMENTAL_API],[test x$enable_experimental_api = xyes]) + AC_CONFIG_FILES([ Makefile apidoc/Makefile @@ -461,7 +477,8 @@ echo " gjs-based tests: $have_gjs dracut: $with_dracut mkinitcpio: $with_mkinitcpio - Static compiler for ostree-prepare-root: $with_static_compiler" + Static compiler for ostree-prepare-root: $with_static_compiler + Experimental API $enable_experimental_api" AS_IF([test x$with_builtin_grub2_mkconfig = xyes], [ echo " builtin grub2-mkconfig (instead of system): $with_builtin_grub2_mkconfig" ], [ diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym new file mode 100644 index 00000000..7ca23a15 --- /dev/null +++ b/src/libostree/libostree-experimental.sym @@ -0,0 +1,32 @@ +/* + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +/* Symbols in this file are added to the build if OSTree is configured with + * --enable-experimental-api. They are not stable or officially supported, and + * might disappear or change in future releases. */ + +/* +LIBOSTREE_2017.6_EXPERIMENTAL { +global: + some_symbol; +} LIBOSTREE_2017.6; +*/ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 55157c07..3ee018b0 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -21,9 +21,17 @@ set -euo pipefail echo '1..2' +if echo "$OSTREE_FEATURES" | grep --quiet --no-messages "experimental"; then + experimental_sym="${G_TEST_SRCDIR}/src/libostree/libostree-experimental.sym" + experimental_sections="${G_TEST_SRCDIR}/apidoc/ostree-experimental-sections.txt" +else + experimental_sym="" + experimental_sections="" +fi + echo "Verifying all expected symbols are actually exported..." -grep ' ostree_[A-Za-z0-9_]*;' ${G_TEST_SRCDIR}/src/libostree/libostree.sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt -eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9_.]*,\1,' |sort -u > found-symbols.txt +grep --no-filename ' ostree_[A-Za-z0-9_]*;' ${G_TEST_SRCDIR}/src/libostree/libostree.sym $experimental_sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt +eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9A-Z_.]*,\1,' |sort -u > found-symbols.txt diff -u expected-symbols.txt found-symbols.txt echo "ok exports" @@ -31,7 +39,7 @@ echo "ok exports" grep -E -v '(ostree_cmd__private__)|(ostree_fetcher_config_flags_get_type)' found-symbols.txt > expected-documented.txt echo "Verifying all public symbols are documented:" -grep '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt |sort -u > found-documented.txt +grep '^ostree_' ${G_TEST_SRCDIR}/apidoc/ostree-sections.txt $experimental_sections |sort -u > found-documented.txt diff -u expected-documented.txt found-documented.txt echo 'ok documented symbols' From eeee5a0a1e509de5d2ea331cfd034e8a19e856c3 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 8 May 2017 16:14:50 +0100 Subject: [PATCH 59/94] libostree: Expose $OSTREE_FEATURES in the pkg-config file This allows consumers of libostree to check at configure time whether it supports the feature they want. Signed-off-by: Philip Withnall Closes: #832 Approved by: cgwalters --- src/libostree/ostree-1.pc.in | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-1.pc.in b/src/libostree/ostree-1.pc.in index d777c597..9a4debce 100644 --- a/src/libostree/ostree-1.pc.in +++ b/src/libostree/ostree-1.pc.in @@ -2,6 +2,7 @@ prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ +features=@OSTREE_FEATURES@ Name: OSTree Description: Git for operating system binaries From 6eac575f21b40310f7d661c0e99fa57dfbe513e5 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Fri, 21 Apr 2017 16:25:23 +0100 Subject: [PATCH 60/94] libostree: Make OstreeRemote a public and internal API Previously it was static to ostree-repo.c. Make it usable throughout libostree so it can be used by an upcoming commit, but also expose the typedef and reference counting functions so that opaque OstreeRemote pointers can be used by user code, in anticipation of exposing more of its API publicly in future. Signed-off-by: Philip Withnall Closes: #832 Approved by: cgwalters --- Makefile-libostree-defines.am | 6 + Makefile-libostree.am | 7 ++ apidoc/ostree-experimental-sections.txt | 6 + src/libostree/libostree-experimental.sym | 5 +- src/libostree/ostree-autocleanups.h | 4 + src/libostree/ostree-remote-private.h | 59 +++++++++ src/libostree/ostree-remote.c | 144 ++++++++++++++++++++++ src/libostree/ostree-remote.h | 56 +++++++++ src/libostree/ostree-repo-private.h | 13 ++ src/libostree/ostree-repo.c | 145 +++++------------------ src/libostree/ostree-types.h | 4 + src/libostree/ostree.h | 3 + 12 files changed, 335 insertions(+), 117 deletions(-) create mode 100644 src/libostree/ostree-remote-private.h create mode 100644 src/libostree/ostree-remote.c create mode 100644 src/libostree/ostree-remote.h diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 1531cf8c..aff7e52b 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -38,6 +38,12 @@ libostree_public_headers = \ src/libostree/ostree-repo-deprecated.h \ $(NULL) +if ENABLE_EXPERIMENTAL_API +libostree_public_headers += \ + src/libostree/ostree-remote.h \ + $(NULL) +endif + # This one is generated via configure.ac, and the gtk-doc # code hence needs to look in the builddir. libostree_public_built_headers = src/libostree/ostree-version.h diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 8d44b61c..86ba0414 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -94,6 +94,8 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-linuxfsutil.c \ src/libostree/ostree-diff.c \ src/libostree/ostree-mutable-tree.c \ + src/libostree/ostree-remote.c \ + src/libostree/ostree-remote-private.h \ src/libostree/ostree-repo.c \ src/libostree/ostree-repo-checkout.c \ src/libostree/ostree-repo-commit.c \ @@ -146,6 +148,11 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-tls-cert-interaction.h \ $(NULL) endif +if !ENABLE_EXPERIMENTAL_API +libostree_1_la_SOURCES += \ + src/libostree/ostree-remote.h \ + $(NULL) +endif libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \ $(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) $(OT_DEP_OPENSSL_CFLAGS) \ diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index e69de29b..790feb33 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -0,0 +1,6 @@ +
+ostree-remote +OstreeRemote +ostree_remote_ref +ostree_remote_unref +
diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 7ca23a15..e2ab4ea8 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -24,9 +24,8 @@ * --enable-experimental-api. They are not stable or officially supported, and * might disappear or change in future releases. */ -/* LIBOSTREE_2017.6_EXPERIMENTAL { global: - some_symbol; + ostree_remote_ref; + ostree_remote_unref; } LIBOSTREE_2017.6; -*/ diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index f030b50e..1fdb50ad 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -59,6 +59,10 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref) G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear) +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #endif G_END_DECLS diff --git a/src/libostree/ostree-remote-private.h b/src/libostree/ostree-remote-private.h new file mode 100644 index 00000000..e207ed4c --- /dev/null +++ b/src/libostree/ostree-remote-private.h @@ -0,0 +1,59 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2011 Colin Walters + * Copyright © 2015 Red Hat, Inc. + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Colin Walters + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "libglnx.h" +#include "ostree-remote.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +struct OstreeRemote { + volatile int ref_count; + char *name; + char *group; /* group name in options */ + char *keyring; /* keyring name (NAME.trustedkeys.gpg) */ + GFile *file; /* NULL if remote defined in repo/config */ + GKeyFile *options; +}; + +G_GNUC_INTERNAL +OstreeRemote *ostree_remote_new (void); + +G_GNUC_INTERNAL +OstreeRemote *ostree_remote_new_from_keyfile (GKeyFile *keyfile, + const gchar *group); + +#if (defined(OSTREE_COMPILATION) || GLIB_CHECK_VERSION(2, 44, 0)) && !defined(OSTREE_ENABLE_EXPERIMENTAL_API) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) +#endif + +G_END_DECLS diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c new file mode 100644 index 00000000..d6298fba --- /dev/null +++ b/src/libostree/ostree-remote.c @@ -0,0 +1,144 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2011 Colin Walters + * Copyright © 2015 Red Hat, Inc. + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Colin Walters + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-remote.h" +#include "ostree-remote-private.h" +#include "ot-keyfile-utils.h" + +/** + * SECTION:remote + * + * The #OstreeRemote structure represents the configuration for a single remote + * repository. Currently, all configuration is handled internally, and + * #OstreeRemote objects are represented by their textual name handle, or by an + * opaque pointer (which can be reference counted if needed). + * + * #OstreeRemote provides configuration for accessing a remote, but does not + * provide the results of accessing a remote, such as information about what + * refs are currently on a remote, or the commits they currently point to. Use + * #OstreeRepo in combination with an #OstreeRemote to query that information. + * + * Since: 2017.6 + */ + +OstreeRemote * +ostree_remote_new (void) +{ + OstreeRemote *remote; + + remote = g_slice_new0 (OstreeRemote); + remote->ref_count = 1; + remote->options = g_key_file_new (); + + return remote; +} + +OstreeRemote * +ostree_remote_new_from_keyfile (GKeyFile *keyfile, + const gchar *group) +{ + g_autoptr(GMatchInfo) match = NULL; + OstreeRemote *remote; + + static gsize regex_initialized; + static GRegex *regex; + + if (g_once_init_enter (®ex_initialized)) + { + regex = g_regex_new ("^remote \"(.+)\"$", 0, 0, NULL); + g_assert (regex); + g_once_init_leave (®ex_initialized, 1); + } + + /* Sanity check */ + g_return_val_if_fail (g_key_file_has_group (keyfile, group), NULL); + + /* If group name doesn't fit the pattern, fail. */ + if (!g_regex_match (regex, group, 0, &match)) + return NULL; + + remote = ostree_remote_new (); + remote->name = g_match_info_fetch (match, 1); + remote->group = g_strdup (group); + remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", remote->name); + + ot_keyfile_copy_group (keyfile, remote->options, group); + + return remote; +} + +/** + * ostree_remote_ref: + * @remote: an #OstreeRemote + * + * Increase the reference count on the given @remote. + * + * Returns: (transfer full): a copy of @remote, for convenience + * Since: 2017.6 + */ +OstreeRemote * +ostree_remote_ref (OstreeRemote *remote) +{ + gint refcount; + g_return_val_if_fail (remote != NULL, NULL); + refcount = g_atomic_int_add (&remote->ref_count, 1); + g_assert (refcount > 0); + return remote; +} + +/** + * ostree_remote_unref: + * @remote: (transfer full): an #OstreeRemote + * + * Decrease the reference count on the given @remote and free it if the + * reference count reaches 0. + * + * Since: 2017.6 + */ +void +ostree_remote_unref (OstreeRemote *remote) +{ + g_return_if_fail (remote != NULL); + g_return_if_fail (remote->ref_count > 0); + + if (g_atomic_int_dec_and_test (&remote->ref_count)) + { + g_clear_pointer (&remote->name, g_free); + g_clear_pointer (&remote->group, g_free); + g_clear_pointer (&remote->keyring, g_free); + g_clear_object (&remote->file); + g_clear_pointer (&remote->options, g_key_file_free); + g_slice_free (OstreeRemote, remote); + } +} diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h new file mode 100644 index 00000000..bf62fd87 --- /dev/null +++ b/src/libostree/ostree-remote.h @@ -0,0 +1,56 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2011 Colin Walters + * Copyright © 2015 Red Hat, Inc. + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Colin Walters + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-types.h" + +G_BEGIN_DECLS + +/** + * OstreeRemote: + * + * This represents the configuration for a single remote repository. Currently, + * remotes can only be passed around as (reference counted) opaque handles. In + * future, more API may be added to create and interrogate them. + * + * Since: 2016.7 + */ +#ifndef OSTREE_ENABLE_EXPERIMENTAL_API +/* This is in ostree-types.h otherwise */ +typedef struct OstreeRemote OstreeRemote; +#endif + +_OSTREE_PUBLIC +OstreeRemote *ostree_remote_ref (OstreeRemote *remote); +_OSTREE_PUBLIC +void ostree_remote_unref (OstreeRemote *remote); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 87e67a23..2a518d4f 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -21,6 +21,7 @@ #pragma once #include "ostree-repo.h" +#include "ostree-remote-private.h" #include "libglnx.h" G_BEGIN_DECLS @@ -350,4 +351,16 @@ gboolean _ostree_repo_update_mtime (OstreeRepo *self, GError **error); +void +_ostree_repo_add_remote (OstreeRepo *self, + OstreeRemote *remote); +OstreeRemote * +_ostree_repo_get_remote (OstreeRepo *self, + const char *name, + GError **error); +OstreeRemote * +_ostree_repo_get_remote_inherited (OstreeRepo *self, + const char *name, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index fd7aa55d..76ae0d9b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -32,6 +32,7 @@ #include #include "ostree-core-private.h" +#include "ostree-remote-private.h" #include "ostree-repo-private.h" #include "ostree-repo-file.h" #include "ostree-repo-file-enumerator.h" @@ -104,94 +105,10 @@ G_DEFINE_TYPE (OstreeRepo, ostree_repo, G_TYPE_OBJECT) #define SYSCONF_REMOTES SHORTENED_SYSCONFDIR "/ostree/remotes.d" -typedef struct { - volatile int ref_count; - char *name; - char *group; /* group name in options */ - char *keyring; /* keyring name (NAME.trustedkeys.gpg) */ - GFile *file; /* NULL if remote defined in repo/config */ - GKeyFile *options; -} OstreeRemote; - -static OstreeRemote * -ost_remote_new (void) -{ - OstreeRemote *remote; - - remote = g_slice_new0 (OstreeRemote); - remote->ref_count = 1; - remote->options = g_key_file_new (); - - return remote; -} - -static OstreeRemote * -ost_remote_new_from_keyfile (GKeyFile *keyfile, - const gchar *group) -{ - g_autoptr(GMatchInfo) match = NULL; - OstreeRemote *remote; - - static gsize regex_initialized; - static GRegex *regex; - - if (g_once_init_enter (®ex_initialized)) - { - regex = g_regex_new ("^remote \"(.+)\"$", 0, 0, NULL); - g_assert (regex); - g_once_init_leave (®ex_initialized, 1); - } - - /* Sanity check */ - g_return_val_if_fail (g_key_file_has_group (keyfile, group), NULL); - - /* If group name doesn't fit the pattern, fail. */ - if (!g_regex_match (regex, group, 0, &match)) - return NULL; - - remote = ost_remote_new (); - remote->name = g_match_info_fetch (match, 1); - remote->group = g_strdup (group); - remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", remote->name); - - ot_keyfile_copy_group (keyfile, remote->options, group); - - return remote; -} - -static OstreeRemote * -ost_remote_ref (OstreeRemote *remote) -{ - gint refcount; - g_return_val_if_fail (remote != NULL, NULL); - refcount = g_atomic_int_add (&remote->ref_count, 1); - g_assert (refcount > 0); - return remote; -} - -static void -ost_remote_unref (OstreeRemote *remote) -{ - g_return_if_fail (remote != NULL); - g_return_if_fail (remote->ref_count > 0); - - if (g_atomic_int_dec_and_test (&remote->ref_count)) - { - g_clear_pointer (&remote->name, g_free); - g_clear_pointer (&remote->group, g_free); - g_clear_pointer (&remote->keyring, g_free); - g_clear_object (&remote->file); - g_clear_pointer (&remote->options, g_key_file_free); - g_slice_free (OstreeRemote, remote); - } -} - -G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeRemote, ost_remote_unref) - -static OstreeRemote * -ost_repo_get_remote (OstreeRepo *self, - const char *name, - GError **error) +OstreeRemote * +_ostree_repo_get_remote (OstreeRepo *self, + const char *name, + GError **error) { OstreeRemote *remote = NULL; @@ -202,7 +119,7 @@ ost_repo_get_remote (OstreeRepo *self, remote = g_hash_table_lookup (self->remotes, name); if (remote != NULL) - ost_remote_ref (remote); + ostree_remote_ref (remote); else g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, "Remote \"%s\" not found", name); @@ -212,19 +129,19 @@ ost_repo_get_remote (OstreeRepo *self, return remote; } -static OstreeRemote * -ost_repo_get_remote_inherited (OstreeRepo *self, - const char *name, - GError **error) +OstreeRemote * +_ostree_repo_get_remote_inherited (OstreeRepo *self, + const char *name, + GError **error) { g_autoptr(OstreeRemote) remote = NULL; g_autoptr(GError) temp_error = NULL; - remote = ost_repo_get_remote (self, name, &temp_error); + remote = _ostree_repo_get_remote (self, name, &temp_error); if (remote == NULL) { if (self->parent_repo != NULL) - return ost_repo_get_remote_inherited (self->parent_repo, name, error); + return _ostree_repo_get_remote_inherited (self->parent_repo, name, error); g_propagate_error (error, g_steal_pointer (&temp_error)); return NULL; @@ -233,9 +150,9 @@ ost_repo_get_remote_inherited (OstreeRepo *self, return g_steal_pointer (&remote); } -static void -ost_repo_add_remote (OstreeRepo *self, - OstreeRemote *remote) +void +_ostree_repo_add_remote (OstreeRepo *self, + OstreeRemote *remote) { g_return_if_fail (self != NULL); g_return_if_fail (remote != NULL); @@ -243,7 +160,7 @@ ost_repo_add_remote (OstreeRepo *self, g_mutex_lock (&self->remotes_lock); - g_hash_table_replace (self->remotes, remote->name, ost_remote_ref (remote)); + g_hash_table_replace (self->remotes, remote->name, ostree_remote_ref (remote)); g_mutex_unlock (&self->remotes_lock); } @@ -308,7 +225,7 @@ ostree_repo_get_remote_option (OstreeRepo *self, return TRUE; } - remote = ost_repo_get_remote (self, remote_name, &temp_error); + remote = _ostree_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { value = g_key_file_get_string (remote->options, remote->group, option_name, &temp_error); @@ -385,7 +302,7 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, return TRUE; } - remote = ost_repo_get_remote (self, remote_name, &temp_error); + remote = _ostree_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { value = g_key_file_get_string_list (remote->options, @@ -461,7 +378,7 @@ ostree_repo_get_remote_boolean_option (OstreeRepo *self, return TRUE; } - remote = ost_repo_get_remote (self, remote_name, &temp_error); + remote = _ostree_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { value = g_key_file_get_boolean (remote->options, remote->group, option_name, &temp_error); @@ -692,7 +609,7 @@ ostree_repo_init (OstreeRepo *self) self->remotes = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify) NULL, - (GDestroyNotify) ost_remote_unref); + (GDestroyNotify) ostree_remote_unref); g_mutex_init (&self->remotes_lock); self->repo_dir_fd = -1; @@ -952,7 +869,7 @@ impl_repo_remote_add (OstreeRepo *self, if (strchr (name, '/') != NULL) return glnx_throw (error, "Invalid character '/' in remote name: %s", name); - g_autoptr(OstreeRemote) remote = ost_repo_get_remote (self, name, NULL); + g_autoptr(OstreeRemote) remote = _ostree_repo_get_remote (self, name, NULL); if (remote != NULL && if_not_exists) { /* Note early return */ @@ -965,7 +882,7 @@ impl_repo_remote_add (OstreeRepo *self, name, remote->file ? gs_file_get_path_cached (remote->file) : "(in config)"); } - remote = ost_remote_new (); + remote = ostree_remote_new (); remote->name = g_strdup (name); remote->group = g_strdup_printf ("remote \"%s\"", name); remote->keyring = g_strdup_printf ("%s.trustedkeys.gpg", name); @@ -1036,7 +953,7 @@ impl_repo_remote_add (OstreeRepo *self, return FALSE; } - ost_repo_add_remote (self, remote); + _ostree_repo_add_remote (self, remote); return TRUE; } @@ -1087,7 +1004,7 @@ impl_repo_remote_delete (OstreeRepo *self, g_autoptr(OstreeRemote) remote = NULL; if (if_exists) { - remote = ost_repo_get_remote (self, name, NULL); + remote = _ostree_repo_get_remote (self, name, NULL); if (!remote) { /* Note early return */ @@ -1095,7 +1012,7 @@ impl_repo_remote_delete (OstreeRepo *self, } } else - remote = ost_repo_get_remote (self, name, error); + remote = _ostree_repo_get_remote (self, name, error); if (remote == NULL) return FALSE; @@ -1412,7 +1329,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, /* First make sure the remote name is valid. */ - remote = ost_repo_get_remote_inherited (self, name, error); + remote = _ostree_repo_get_remote_inherited (self, name, error); if (remote == NULL) goto out; @@ -1624,7 +1541,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, out: if (remote != NULL) - ost_remote_unref (remote); + ostree_remote_unref (remote); if (source_tmp_dir != NULL) (void) glnx_shutil_rm_rf_at (AT_FDCWD, source_tmp_dir, NULL, NULL); @@ -1857,7 +1774,7 @@ add_remotes_from_keyfile (OstreeRepo *self, { OstreeRemote *remote; - remote = ost_remote_new_from_keyfile (keyfile, groups[ii]); + remote = ostree_remote_new_from_keyfile (keyfile, groups[ii]); if (remote != NULL) { @@ -1888,7 +1805,7 @@ add_remotes_from_keyfile (OstreeRepo *self, out: while (!g_queue_is_empty (&queue)) - ost_remote_unref (g_queue_pop_head (&queue)); + ostree_remote_unref (g_queue_pop_head (&queue)); g_mutex_unlock (&self->remotes_lock); @@ -4213,7 +4130,7 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, OstreeRemote *remote; g_autoptr(GFile) file = NULL; - remote = ost_repo_get_remote_inherited (self, remote_name, error); + remote = _ostree_repo_get_remote_inherited (self, remote_name, error); if (remote == NULL) return NULL; @@ -4232,7 +4149,7 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, if (gpgkeypath) _ostree_gpg_verifier_add_key_ascii_file (verifier, gpgkeypath); - ost_remote_unref (remote); + ostree_remote_unref (remote); } if (add_global_keyring_dir) diff --git a/src/libostree/ostree-types.h b/src/libostree/ostree-types.h index f6aea0f0..61ac4283 100644 --- a/src/libostree/ostree-types.h +++ b/src/libostree/ostree-types.h @@ -38,4 +38,8 @@ typedef struct OstreeSysrootUpgrader OstreeSysrootUpgrader; typedef struct OstreeMutableTree OstreeMutableTree; typedef struct OstreeRepoFile OstreeRepoFile; +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +typedef struct OstreeRemote OstreeRemote; +#endif + G_END_DECLS diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index eb4ed8d3..5d1ac1e1 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -24,6 +24,9 @@ #include #include #include +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#include +#endif #include #include #include From 50f73cbac35be97fd5895531e295d05dabaa8ed9 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Sun, 7 May 2017 20:57:53 +0100 Subject: [PATCH 61/94] build: Add -C arguments to some git invocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This moves the build system a little closer towards being safe for builddir ≠ srcdir. Signed-off-by: Philip Withnall Closes: #832 Approved by: cgwalters --- Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 78cd6653..be505522 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,7 +19,7 @@ include Makefile-decls.am shortened_sysconfdir = $$(echo "$(sysconfdir)" | sed -e 's|^$(prefix)||' -e 's|^/||') -OSTREE_GITREV=$(shell if command -v git >/dev/null 2>&1 && test -d $(srcdir)/.git; then git describe --abbrev=42 --tags --always HEAD; fi) +OSTREE_GITREV=$(shell if command -v git >/dev/null 2>&1 && test -d $(srcdir)/.git; then git -C $(srcdir) describe --abbrev=42 --tags --always HEAD; fi) ACLOCAL_AMFLAGS = -I buildutil -I libglnx ${ACLOCAL_FLAGS} AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ @@ -113,11 +113,11 @@ include Makefile-boot.am include Makefile-man.am release-tag: - git tag -m "Release $(VERSION)" v$(VERSION) + git -C $(srcdir) tag -m "Release $(VERSION)" v$(VERSION) embed_dependency=tar -C $(srcdir) --append --exclude='.git/*' --transform="s,^embedded-dependencies/,ostree-embeddeps-$${GITVERSION}/embedded-dependencies/," --file=$${TARFILE_TMP} -git_version_rpm = $$(git describe | sed -e 's,-,\.,g' -e 's,^v,,') +git_version_rpm = $$(git -C $(srcdir) describe | sed -e 's,-,\.,g' -e 's,^v,,') release-tarball-embedded: set -x; \ From 052ba81c03dff0da45e234b8b60d25f0f06dec63 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 9 May 2017 09:26:11 -0400 Subject: [PATCH 62/94] utils/checksum: Port to new code style Just happened to be reading this code, it's an easy port. Closes: #842 Approved by: jlebon --- src/libotutil/ot-checksum-utils.c | 58 ++++++++++--------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c index 39417044..eb0185bd 100644 --- a/src/libotutil/ot-checksum-utils.c +++ b/src/libotutil/ot-checksum-utils.c @@ -47,7 +47,7 @@ ot_csum_from_gchecksum (GChecksum *checksum) { guchar *ret = g_malloc (32); gsize len = 32; - + g_checksum_get_digest (checksum, ret, &len); g_assert (len == 32); return ret; @@ -62,13 +62,11 @@ ot_gio_write_update_checksum (GOutputStream *out, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - if (out) { if (!g_output_stream_write_all (out, data, len, out_bytes_written, cancellable, error)) - goto out; + return FALSE; } else if (out_bytes_written) { @@ -77,10 +75,7 @@ ot_gio_write_update_checksum (GOutputStream *out, if (checksum) g_checksum_update (checksum, data, len); - - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -90,8 +85,6 @@ ot_gio_splice_update_checksum (GOutputStream *out, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_return_val_if_fail (out != NULL || checksum != NULL, FALSE); if (checksum != NULL) @@ -101,24 +94,25 @@ ot_gio_splice_update_checksum (GOutputStream *out, do { if (!g_input_stream_read_all (in, buf, sizeof(buf), &bytes_read, cancellable, error)) - goto out; + return FALSE; if (!ot_gio_write_update_checksum (out, buf, bytes_read, &bytes_written, checksum, cancellable, error)) - goto out; + return FALSE; } while (bytes_read > 0); } else if (out != NULL) { if (g_output_stream_splice (out, in, 0, cancellable, error) < 0) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } +/* Copy @in to @out, return in @out_csum the binary checksum for + * all data read. + */ gboolean ot_gio_splice_get_checksum (GOutputStream *out, GInputStream *in, @@ -126,22 +120,14 @@ ot_gio_splice_get_checksum (GOutputStream *out, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - GChecksum *checksum = NULL; - g_autofree guchar *ret_csum = NULL; - - checksum = g_checksum_new (G_CHECKSUM_SHA256); + g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256); if (!ot_gio_splice_update_checksum (out, in, checksum, cancellable, error)) - goto out; + return FALSE; - ret_csum = ot_csum_from_gchecksum (checksum); - - ret = TRUE; + g_autofree guchar *ret_csum = ot_csum_from_gchecksum (checksum); ot_transfer_out_value (out_csum, &ret_csum); - out: - g_clear_pointer (&checksum, (GDestroyNotify) g_checksum_free); - return ret; + return TRUE; } gboolean @@ -162,21 +148,13 @@ ot_checksum_file_at (int dfd, GCancellable *cancellable, GError **error) { - GChecksum *checksum = NULL; - char *ret = NULL; g_autoptr(GInputStream) in = NULL; - if (!ot_openat_read_stream (dfd, path, TRUE, &in, cancellable, error)) - goto out; - - checksum = g_checksum_new (checksum_type); + return FALSE; + g_autoptr(GChecksum) checksum = g_checksum_new (checksum_type); if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error)) - goto out; - - ret = g_strdup (g_checksum_get_string (checksum)); - out: - g_clear_pointer (&checksum, (GDestroyNotify) g_checksum_free); - return ret; + return FALSE; + return g_strdup (g_checksum_get_string (checksum)); } From 86963334bdef83ddd9f55dda33e3b681851b9cc5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 May 2017 16:59:14 -0400 Subject: [PATCH 63/94] fsck: Check for refs missing corresponding commit Just doing this one quickly since it was easy. Closes: https://github.com/ostreedev/ostree/issues/831 Closes: #841 Approved by: jlebon --- src/ostree/ot-builtin-fsck.c | 22 ++++++++++++++++++++++ tests/test-corruption.sh | 17 ++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c index 2f154cff..7519d6db 100644 --- a/src/ostree/ot-builtin-fsck.c +++ b/src/ostree/ot-builtin-fsck.c @@ -246,6 +246,7 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** gpointer key, value; gboolean found_corruption = FALSE; guint n_partial = 0; + g_autoptr(GHashTable) all_refs = NULL; g_autoptr(GHashTable) objects = NULL; g_autoptr(GHashTable) commits = NULL; g_autoptr(GPtrArray) tombstones = NULL; @@ -254,6 +255,27 @@ ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError ** if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) goto out; + if (!opt_quiet) + g_print ("Validating refs...\n"); + + /* Validate that the commit for each ref is available */ + if (!ostree_repo_list_refs (repo, NULL, &all_refs, + cancellable, error)) + return FALSE; + g_hash_table_iter_init (&hash_iter, all_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *refname = key; + const char *checksum = value; + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, + checksum, &commit, error)) + { + g_prefix_error (error, "Loading commit for ref %s: ", refname); + goto out; + } + } + if (!opt_quiet) g_print ("Enumerating objects...\n"); diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index ef0e94ef..8e2aba56 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -19,10 +19,12 @@ set -euo pipefail -echo "1..2" +echo "1..3" . $(dirname $0)/libtest.sh +cd ${test_tmpdir} +rm repo files -rf setup_test_repository "bare" $OSTREE checkout test2 checkout-test2 cd checkout-test2 @@ -34,6 +36,8 @@ $OSTREE fsck -q echo "ok chmod" cd ${test_tmpdir} +rm repo files -rf +setup_test_repository "bare" rm checkout-test2 -rf $OSTREE checkout test2 checkout-test2 cd checkout-test2 @@ -41,3 +45,14 @@ chmod o+x firstfile $OSTREE fsck -q --delete && (echo 1>&2 "fsck unexpectedly succeeded"; exit 1) echo "ok chmod" + +cd ${test_tmpdir} +rm repo files -rf +setup_test_repository "bare" +find repo/ -name '*.commit' -delete +if $OSTREE fsck -q 2>err.txt; then + assert_not_reached "fsck unexpectedly succeeded" +fi +assert_file_has_content_literal err.txt "Loading commit for ref test2: No such metadata object" + +echo "ok missing commit" From 2800d176bc300f864a1dc16ddfc63c78356c3061 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 May 2017 11:20:32 -0400 Subject: [PATCH 64/94] tests: For installed, s/test-/itest-/ to avoid in-tree name clashes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I want to migrate `test-pull-many.sh` → `itest-pull.sh`, hence not conflicting with the unit test `test-pull.sh. Closes: #840 Approved by: jlebon --- tests/installed/{test-bare-root.sh => itest-bare-root.sh} | 0 .../{test-deploy-selinux.sh => itest-deploy-selinux.sh} | 0 tests/installed/run.sh | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/installed/{test-bare-root.sh => itest-bare-root.sh} (100%) rename tests/installed/{test-deploy-selinux.sh => itest-deploy-selinux.sh} (100%) diff --git a/tests/installed/test-bare-root.sh b/tests/installed/itest-bare-root.sh similarity index 100% rename from tests/installed/test-bare-root.sh rename to tests/installed/itest-bare-root.sh diff --git a/tests/installed/test-deploy-selinux.sh b/tests/installed/itest-deploy-selinux.sh similarity index 100% rename from tests/installed/test-deploy-selinux.sh rename to tests/installed/itest-deploy-selinux.sh diff --git a/tests/installed/run.sh b/tests/installed/run.sh index 3c60a6e2..02e1a7f9 100755 --- a/tests/installed/run.sh +++ b/tests/installed/run.sh @@ -3,7 +3,7 @@ set -xeuo pipefail dn=$(dirname $0) -for tn in ${dn}/test-*.sh; do +for tn in ${dn}/itest-*.sh; do echo Executing: ${tn} ${tn} done From 48d2637e98a6f2272dd13e3406060f2bef1c48f1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 May 2017 13:37:50 -0400 Subject: [PATCH 65/94] tests: Migrate test-pull-many.sh to installed on FAH `test-pull-many.sh` is was just too slow to be a unit test. Generating a bunch of files via shell is slow, the delta generation is slow, etc. Every developer doesn't need to run it every time. Somewhat address this by converting it into our installed test framework, which moves it out of the developer fast paths. Another advantage to this is that we can simply reuse the FAH tree content rather than synthesizing new bits each time. Closes: #840 Approved by: jlebon --- Makefile-tests.am | 1 - tests/installed/itest-pull.sh | 27 ++++++++++++++++++ tests/installed/libinsttest.sh | 43 ++++++++++++++++++++++++++++ tests/test-pull-many.sh | 52 ---------------------------------- 4 files changed, 70 insertions(+), 53 deletions(-) create mode 100755 tests/installed/itest-pull.sh delete mode 100755 tests/test-pull-many.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 14e30b46..335ba9b7 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -77,7 +77,6 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-resume.sh \ tests/test-pull-repeated.sh \ tests/test-pull-untrusted.sh \ - tests/test-pull-many.sh \ tests/test-pull-override-url.sh \ tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ diff --git a/tests/installed/itest-pull.sh b/tests/installed/itest-pull.sh new file mode 100755 index 00000000..e3125f4c --- /dev/null +++ b/tests/installed/itest-pull.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Using the host ostree, test HTTP pulls + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +test_tmpdir=$(prepare_tmpdir) +trap _tmpdir_cleanup EXIT + +# Take the host's ostree, and make it archive +cd /var/srv +rm repo bare-repo -rf +mkdir repo +ostree --repo=repo init --mode=archive +echo -e '[archive]\nzlib-level=1\n' >> repo/config +host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,') +ostree --repo=repo pull-local /ostree/repo ${host_commit} +ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref} + +run_tmp_webserver $(pwd)/repo +# Now test pulling via HTTP (no deltas) to a new bare-user repo +ostree --repo=bare-repo init --mode=bare-user +ostree --repo=bare-repo remote add origin --set=gpg-verify=false $(cat ${test_tmpdir}/httpd-address) +ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref} diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh index 4f72b651..759c95f2 100644 --- a/tests/installed/libinsttest.sh +++ b/tests/installed/libinsttest.sh @@ -20,6 +20,49 @@ dn=$(dirname $0) . ${dn}/libtest-core.sh +# Copy of bits from tap-test +test_tmpdir= +function _tmpdir_cleanup () { + if test -n "${test_tmpdir}" && test -f ${test_tmpdir}/.testtmp; then + rm "${test_tmpdir}" -rf + fi +} +prepare_tmpdir() { + test_tmpdir=$(mktemp -d) + touch ${test_tmpdir}/.testtmp + cd ${test_tmpdir} + echo ${test_tmpdir} +} + +# This is copied from flatpak/flatpak/tests/test-webserver.sh +run_tmp_webserver() { + dir=$1 + + test -n ${test_tmpdir} + + cd ${dir} + env PYTHONUNBUFFERED=1 setsid python -m SimpleHTTPServer 0 &>${test_tmpdir}/httpd-output & + cd - + child_pid=$! + + for x in $(seq 50); do + # Snapshot the output + cp ${test_tmpdir}/httpd-output{,.tmp} + # If it's non-empty, see whether it matches our regexp + if test -s ${test_tmpdir}/httpd-output.tmp; then + sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port + if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then + # If so, we've successfully extracted the port + break + fi + fi + sleep 0.1 + done + port=$(cat ${test_tmpdir}/httpd-port) + echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address + echo "$child_pid" > ${test_tmpdir}/httpd-pid +} + # Determine our origin refspec - we'll use this as a test base rpmostree=$(which rpm-ostree 2>/dev/null) if test -z "${rpmostree}"; then diff --git a/tests/test-pull-many.sh b/tests/test-pull-many.sh deleted file mode 100755 index 73245a6b..00000000 --- a/tests/test-pull-many.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2017 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -set -euo pipefail - -. $(dirname $0)/libtest.sh - -setup_exampleos_repo - -echo '1..3' - -cd ${test_tmpdir} -set -x - -echo "$(date): Pulling content..." -rev=$(${CMD_PREFIX} ostree --repo=ostree-srv/exampleos/repo rev-parse ${REF}) -${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin ${REF} -${CMD_PREFIX} ostree --repo=repo fsck -assert_streq ${rev} $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF}) - -echo "ok without deltas" - -previous=$(${CMD_PREFIX} ostree --repo=repo rev-parse ${rev}^) -rm repo/refs/{heads,remotes}/* -rf -${CMD_PREFIX} ostree --repo=repo prune --refs-only -${CMD_PREFIX} ostree --repo=repo pull origin ${REF}@${previous} -${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${REF} > output.txt -assert_file_has_content output.txt 'Delta update: 0/1 parts, 0 bytes/1.[012] MB, 1.[345] MB total uncompressed' - -echo "ok delta dry-run" - -${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${REF} -assert_streq $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF}) ${rev} -${CMD_PREFIX} ostree --repo=repo fsck - -echo "ok" From af7fed94ed901dd1b57962c1e6510dc93c1954ec Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 8 May 2017 16:44:42 -0400 Subject: [PATCH 66/94] ci: Extend FAH rootfs for installed tests These at the moment aren't in a container, and may need space. In the future overlay2 will help here, we can more easily extend the rootfs. Closes: #840 Approved by: jlebon --- .redhat-ci.yml | 2 +- tests/installed/fah-prep.sh | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100755 tests/installed/fah-prep.sh diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 818a072a..e3eb0f87 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -132,7 +132,7 @@ build: tests: - make install DESTDIR=$(pwd)/insttree - rsync -rl -e 'ssh -o User=root' . vmcheck:ostree/ - - ssh root@vmcheck 'ostree admin unlock && rsync -rlv ./ostree/insttree/usr/ /usr/ && ./ostree/tests/installed/run.sh' + - ssh root@vmcheck './ostree/tests/installed/fah-prep.sh && ostree admin unlock && rsync -rlv ./ostree/insttree/usr/ /usr/ && ./ostree/tests/installed/run.sh' artifacts: - test-suite.log diff --git a/tests/installed/fah-prep.sh b/tests/installed/fah-prep.sh new file mode 100755 index 00000000..0db4d15e --- /dev/null +++ b/tests/installed/fah-prep.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -xeuo pipefail +# If we're using devmapper, expand the root +if lvm lvs atomicos/docker-pool &>/dev/null; then + systemctl stop docker + lvm lvremove -f atomicos/docker-pool +fi +lvm lvextend -r -l +100%FREE atomicos/root From 7c88161044b73c754e59e9fcb8855938eef9ea25 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 1 May 2017 18:02:27 -0400 Subject: [PATCH 67/94] ci: More flatpak ci fixes We need our `make install` to override the ostree RPM, so do it all in one txn. This sort of thing is where a more rigorous model like rdgo/gcontinuous use becomes better, but we'll hack it with shell for now. Closes: #824 Approved by: jlebon --- .redhat-ci.yml | 2 + ...-Fix-race-condition-in-tmp-webserver.patch | 44 +++++++++++++++++++ ci/flatpak.sh | 26 ++++++----- 3 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 ci/0001-tests-Fix-race-condition-in-tmp-webserver.patch diff --git a/.redhat-ci.yml b/.redhat-ci.yml index e3eb0f87..97bbcf87 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -153,6 +153,8 @@ required: false # docker --privileged run. host: distro: fedora/25/atomic + specs: + ram: 4096 # build-bundle is a static delta, which needs RAM right now tests: - docker run --rm --privileged -v $(pwd):/srv/code registry.fedoraproject.org/fedora:25 /bin/sh -c "cd /srv/code && ./ci/flatpak.sh" diff --git a/ci/0001-tests-Fix-race-condition-in-tmp-webserver.patch b/ci/0001-tests-Fix-race-condition-in-tmp-webserver.patch new file mode 100644 index 00000000..dc66425f --- /dev/null +++ b/ci/0001-tests-Fix-race-condition-in-tmp-webserver.patch @@ -0,0 +1,44 @@ +From 6197e6922e3ba3c8881733a6a3253e8ae12eb538 Mon Sep 17 00:00:00 2001 +From: Colin Walters +Date: Fri, 5 May 2017 16:36:04 -0400 +Subject: [PATCH] tests: Fix race condition in tmp webserver + +I was seeing this when trying to run flatpak's tests in ostree's CI: +https://github.com/ostreedev/ostree/pull/824 + +The race here is that the python process can still be writing to the output +while sed is reading it, and hence we'll find a difference on the next line. +Fix this by making a tmp copy of the file, which then both sed and cmp will +read consistently. + +I'm not *entirely* sure this will fix the problem as I couldn't easily reproduce +the race locally, but I believe it at least fixes *a* race. +--- + tests/test-webserver.sh | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/tests/test-webserver.sh b/tests/test-webserver.sh +index 3291b06..2964ce9 100755 +--- a/tests/test-webserver.sh ++++ b/tests/test-webserver.sh +@@ -10,9 +10,15 @@ env PYTHONUNBUFFERED=1 setsid python -m SimpleHTTPServer 0 >${test_tmpdir}/httpd + child_pid=$! + + for x in $(seq 50); do +- sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) \.\.\.,\1,' < ${test_tmpdir}/httpd-output > ${test_tmpdir}/httpd-port +- if ! cmp ${test_tmpdir}/httpd-output ${test_tmpdir}/httpd-port 1>/dev/null; then +- break ++ # Snapshot the output ++ cp ${test_tmpdir}/httpd-output{,.tmp} ++ # If it's non-empty, see whether it matches our regexp ++ if test -s ${test_tmpdir}/httpd-output.tmp; then ++ sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port ++ if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then ++ # If so, we've successfully extracted the port ++ break ++ fi + fi + sleep 0.1 + done +-- +2.9.3 diff --git a/ci/flatpak.sh b/ci/flatpak.sh index fee738aa..7d98ff05 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -12,23 +12,27 @@ build() { codedir=$(pwd) # Core prep -dnf -y install dnf-plugins-core -dnf install -y @buildsys-build -dnf install -y 'dnf-command(builddep)' +yum -y install dnf-plugins-core @buildsys-build 'dnf-command(builddep)' +# build+install ostree, and build deps for both, so that our +# make install overrides the ostree via rpm +dnf builddep -y ostree flatpak +yum -y install flatpak && rpm -e flatpak +# we use yaml below +yum -y install python3-PyYAML -# build+install ostree -dnf builddep -y ostree -build -make install +# Now get flatpak's deps from rhci file tmpd=$(mktemp -d) cd ${tmpd} # Frozen to a tag for now on general principle git clone --recursive --depth=1 -b 0.9.3 https://github.com/flatpak/flatpak cd flatpak -dnf builddep -y flatpak -# And runtime deps -dnf install -y flatpak && rpm -e flatpak -dnf install -y which attr fuse parallel # for the test suite +python3 -c 'import yaml; y = list(yaml.load_all(open(".redhat-ci.yml")))[0]; print("\0".join(y["packages"]))' | xargs -0 yum install -y +# back to ostree and build +cd ${codedir} +build +make install +cd ${tmpd}/flatpak +patch -p1 < ${codedir}/ci/*.patch build # We want to capture automake results from flatpak cleanup() { From bf1a994d85679a6b6690778a851866de13fafd3f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 9 May 2017 11:52:20 -0400 Subject: [PATCH 68/94] =?UTF-8?q?ci:=20Move=20travis=20scripts=20from=20te?= =?UTF-8?q?sts/=20=E2=86=92=20ci/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I think tests/ should be just that, ci/ is separate. Also rename the files to include "travis" since that's what we use them for right now. Closes: #843 Approved by: jlebon --- .travis.yml | 4 ++-- tests/ci-Dockerfile.in => ci/travis-Dockerfile.in | 4 ++-- tests/ci-build.sh => ci/travis-build.sh | 4 ++-- tests/ci-install.sh => ci/travis-install.sh | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) rename tests/ci-Dockerfile.in => ci/travis-Dockerfile.in (75%) rename tests/ci-build.sh => ci/travis-build.sh (96%) rename tests/ci-install.sh => ci/travis-install.sh (95%) diff --git a/.travis.yml b/.travis.yml index a021592c..140e7ada 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,8 @@ env: - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial script: - - tests/ci-install.sh - - ci_parallel=2 ci_sudo=yes tests/ci-build.sh + - ci/travis-install.sh + - ci_parallel=2 ci_sudo=yes ci/travis-build.sh notifications: email: false diff --git a/tests/ci-Dockerfile.in b/ci/travis-Dockerfile.in similarity index 75% rename from tests/ci-Dockerfile.in rename to ci/travis-Dockerfile.in index 0b56b431..df999e78 100644 --- a/tests/ci-Dockerfile.in +++ b/ci/travis-Dockerfile.in @@ -1,8 +1,8 @@ FROM @ci_docker@ ENV container docker -ADD tests/ci-install.sh /ci-install.sh -RUN ci_suite="@ci_suite@" ci_distro="@ci_distro@" ci_in_docker=yes /ci-install.sh +ADD ci/travis-install.sh /travis-install.sh +RUN ci_suite="@ci_suite@" ci_distro="@ci_distro@" ci_in_docker=yes /travis-install.sh ADD . /home/user/ostree RUN chown -R user:user /home/user/ostree diff --git a/tests/ci-build.sh b/ci/travis-build.sh similarity index 96% rename from tests/ci-build.sh rename to ci/travis-build.sh index e310ed0f..885e3ce2 100755 --- a/tests/ci-build.sh +++ b/ci/travis-build.sh @@ -28,7 +28,7 @@ set -x NULL= # ci_docker: -# If non-empty, this is the name of a Docker image. ci-install.sh will +# If non-empty, this is the name of a Docker image. travis-install.sh will # fetch it with "docker pull" and use it as a base for a new Docker image # named "ci-image" in which we will do our testing. # @@ -66,7 +66,7 @@ if [ -n "$ci_docker" ]; then --env=ci_configopts="${ci_configopts}" \ --privileged \ ci-image \ - tests/ci-build.sh + ci/travis-build.sh fi maybe_fail_tests () { diff --git a/tests/ci-install.sh b/ci/travis-install.sh similarity index 95% rename from tests/ci-install.sh rename to ci/travis-install.sh index d927d962..5d3d5a91 100755 --- a/tests/ci-install.sh +++ b/ci/travis-install.sh @@ -33,13 +33,13 @@ NULL= : "${ci_distro:=debian}" # ci_docker: -# If non-empty, this is the name of a Docker image. ci-install.sh will +# If non-empty, this is the name of a Docker image. travis-install.sh will # fetch it with "docker pull" and use it as a base for a new Docker image # named "ci-image" in which we will do our testing. : "${ci_docker:=}" # ci_in_docker: -# Used internally by ci-install.sh. If yes, we are inside the Docker image +# Used internally by travis-install.sh. If yes, we are inside the Docker image # (ci_docker is empty in this case). : "${ci_in_docker:=no}" @@ -64,7 +64,7 @@ if [ -n "$ci_docker" ]; then -e "s/@ci_distro@/${ci_distro}/" \ -e "s/@ci_docker@/${ci_docker}/" \ -e "s/@ci_suite@/${ci_suite}/" \ - < tests/ci-Dockerfile.in > Dockerfile + < ci/travis-Dockerfile.in > Dockerfile exec docker build -t ci-image . fi From 18c5947c5f2bc1d104d92d0fc42b52d6169bfdea Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 9 May 2017 13:39:51 -0400 Subject: [PATCH 69/94] diff: Port some to new code style Continuing to chip away at this. Using `g_file_enumerator_iterate()` here helps notably. I started on the much bigger `ostree_diff_dirs_with_options()` but it's a lot messier - for later. Closes: #844 Approved by: jlebon --- src/libostree/ostree-diff.c | 70 ++++++++++++++----------------------- 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/src/libostree/ostree-diff.c b/src/libostree/ostree-diff.c index b428fb44..f21b0302 100644 --- a/src/libostree/ostree-diff.c +++ b/src/libostree/ostree-diff.c @@ -35,9 +35,7 @@ get_file_checksum (OstreeDiffFlags flags, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *ret_checksum = NULL; - g_autofree guchar *csum = NULL; if (OSTREE_IS_REPO_FILE (f)) { @@ -52,27 +50,26 @@ get_file_checksum (OstreeDiffFlags flags, { if (!glnx_dfd_name_get_all_xattrs (AT_FDCWD, gs_file_get_path_cached (f), &xattrs, cancellable, error)) - goto out; + return FALSE; } if (g_file_info_get_file_type (f_info) == G_FILE_TYPE_REGULAR) { in = (GInputStream*)g_file_read (f, cancellable, error); if (!in) - goto out; + return FALSE; } + g_autofree guchar *csum = NULL; if (!ostree_checksum_file_from_input (f_info, xattrs, in, OSTREE_OBJECT_TYPE_FILE, &csum, cancellable, error)) - goto out; + return FALSE; ret_checksum = ostree_checksum_from_bytes (csum); } - ret = TRUE; ot_transfer_out_value(out_checksum, &ret_checksum); - out: - return ret; + return TRUE; } OstreeDiffItem * @@ -130,28 +127,22 @@ diff_files (OstreeDiffFlags flags, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *checksum_a = NULL; g_autofree char *checksum_b = NULL; - OstreeDiffItem *ret_item = NULL; - if (!get_file_checksum (flags, a, a_info, &checksum_a, cancellable, error)) - goto out; + return FALSE; if (!get_file_checksum (flags, b, b_info, &checksum_b, cancellable, error)) - goto out; + return FALSE; + g_autoptr(OstreeDiffItem) ret_item = NULL; if (strcmp (checksum_a, checksum_b) != 0) { ret_item = diff_item_new (a, a_info, b, b_info, checksum_a, checksum_b); } - ret = TRUE; ot_transfer_out_value(out_item, &ret_item); - out: - if (ret_item) - ostree_diff_item_unref (ret_item); - return ret; + return TRUE; } static gboolean @@ -160,47 +151,38 @@ diff_add_dir_recurse (GFile *d, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - GError *temp_error = NULL; - g_autoptr(GFileEnumerator) dir_enum = NULL; - g_autoptr(GFile) child = NULL; - g_autoptr(GFileInfo) child_info = NULL; - - dir_enum = g_file_enumerate_children (d, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); + g_autoptr(GFileEnumerator) dir_enum = + g_file_enumerate_children (d, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, + error); if (!dir_enum) - goto out; + return FALSE; - while ((child_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL) + while (TRUE) { + GFileInfo *child_info; const char *name; + if (!g_file_enumerator_iterate (dir_enum, &child_info, NULL, + cancellable, error)) + return FALSE; + if (child_info == NULL) + break; + name = g_file_info_get_name (child_info); - g_clear_object (&child); - child = g_file_get_child (d, name); - + g_autoptr(GFile) child = g_file_get_child (d, name); g_ptr_array_add (added, g_object_ref (child)); if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY) { if (!diff_add_dir_recurse (child, added, cancellable, error)) - goto out; + return FALSE; } - - g_clear_object (&child_info); - } - if (temp_error != NULL) - { - g_propagate_error (error, temp_error); - goto out; } - ret = TRUE; - out: - return ret; + return TRUE; } /** From 05fda71cb1f8f0709a7e8c090ed6490aa8d3c089 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 9 May 2017 15:55:03 -0400 Subject: [PATCH 70/94] sysroot: More porting to new code style This isn't all of this file yet, just doing another chunk. Closes: #845 Approved by: jlebon --- src/libostree/ostree-sysroot.c | 226 +++++++++++---------------------- 1 file changed, 75 insertions(+), 151 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index b9f6c66e..67f940a8 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -587,60 +587,54 @@ parse_deployment (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const char *relative_boot_link; - glnx_unref_object OstreeDeployment *ret_deployment = NULL; + if (!ensure_sysroot_fd (self, error)) + return FALSE; + int entry_boot_version; - int treebootserial = -1; - int deployserial = -1; g_autofree char *osname = NULL; g_autofree char *bootcsum = NULL; - g_autofree char *treecsum = NULL; - glnx_fd_close int deployment_dfd = -1; - const char *deploy_basename; - g_autofree char *treebootserial_target = NULL; - g_autoptr(GKeyFile) origin = NULL; - g_autofree char *unlocked_development_path = NULL; - struct stat stbuf; - - if (!ensure_sysroot_fd (self, error)) - goto out; - + int treebootserial = -1; if (!parse_bootlink (boot_link, &entry_boot_version, &osname, &bootcsum, &treebootserial, error)) - goto out; + return FALSE; - relative_boot_link = boot_link; + const char *relative_boot_link = boot_link; if (*relative_boot_link == '/') relative_boot_link++; - treebootserial_target = glnx_readlinkat_malloc (self->sysroot_fd, relative_boot_link, - cancellable, error); + g_autofree char *treebootserial_target = + glnx_readlinkat_malloc (self->sysroot_fd, relative_boot_link, + cancellable, error); if (!treebootserial_target) - goto out; - - deploy_basename = glnx_basename (treebootserial_target); + return FALSE; + const char *deploy_basename = glnx_basename (treebootserial_target); + g_autofree char *treecsum = NULL; + int deployserial = -1; if (!_ostree_sysroot_parse_deploy_path_name (deploy_basename, &treecsum, &deployserial, error)) - goto out; + return FALSE; + glnx_fd_close int deployment_dfd = -1; if (!glnx_opendirat (self->sysroot_fd, relative_boot_link, TRUE, &deployment_dfd, error)) - goto out; + return FALSE; + g_autoptr(GKeyFile) origin = NULL; if (!parse_origin (self, deployment_dfd, deploy_basename, &origin, cancellable, error)) - goto out; + return FALSE; - ret_deployment = ostree_deployment_new (-1, osname, treecsum, deployserial, - bootcsum, treebootserial); + glnx_unref_object OstreeDeployment *ret_deployment + = ostree_deployment_new (-1, osname, treecsum, deployserial, + bootcsum, treebootserial); if (origin) ostree_deployment_set_origin (ret_deployment, origin); ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_NONE; - unlocked_development_path = get_unlocked_development_path (ret_deployment); + g_autofree char *unlocked_development_path = get_unlocked_development_path (ret_deployment); + struct stat stbuf; if (lstat (unlocked_development_path, &stbuf) == 0) ret_deployment->unlocked = OSTREE_DEPLOYMENT_UNLOCKED_DEVELOPMENT; else @@ -657,11 +651,9 @@ parse_deployment (OstreeSysroot *self, g_debug ("Deployment %s.%d unlocked=%d", treecsum, deployserial, ret_deployment->unlocked); - ret = TRUE; if (out_deployment) *out_deployment = g_steal_pointer (&ret_deployment); - out: - return ret; + return TRUE; } static char * @@ -747,29 +739,21 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - guint i; - int bootversion = 0; - int subbootversion = 0; - struct stat stbuf; - g_autoptr(GPtrArray) boot_loader_configs = NULL; - g_autoptr(GPtrArray) deployments = NULL; - if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; + int bootversion = 0; if (!read_current_bootversion (self, &bootversion, cancellable, error)) - goto out; + return FALSE; + int subbootversion = 0; if (!_ostree_sysroot_read_current_subbootversion (self, bootversion, &subbootversion, cancellable, error)) - goto out; + return FALSE; + struct stat stbuf; if (fstatat (self->sysroot_fd, "ostree/deploy", &stbuf, 0) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fstatat"); if (out_changed) { @@ -777,8 +761,8 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, self->loaded_ts.tv_nsec == stbuf.st_mtim.tv_nsec) { *out_changed = FALSE; - ret = TRUE; - goto out; + /* Note early return */ + return TRUE; } } @@ -787,23 +771,24 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, self->bootversion = -1; self->subbootversion = -1; + g_autoptr(GPtrArray) boot_loader_configs = NULL; if (!_ostree_sysroot_read_boot_loader_configs (self, bootversion, &boot_loader_configs, cancellable, error)) - goto out; + return FALSE; - deployments = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + g_autoptr(GPtrArray) deployments = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - for (i = 0; i < boot_loader_configs->len; i++) + for (guint i = 0; i < boot_loader_configs->len; i++) { OstreeBootconfigParser *config = boot_loader_configs->pdata[i]; if (!list_deployments_process_one_boot_entry (self, config, deployments, cancellable, error)) - goto out; + return FALSE; } g_ptr_array_sort (deployments, compare_deployments_by_boot_loader_version_reversed); - for (i = 0; i < deployments->len; i++) + for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; ostree_deployment_set_index (deployment, i); @@ -811,7 +796,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!find_booted_deployment (self, deployments, &self->booted_deployment, cancellable, error)) - goto out; + return FALSE; self->bootversion = bootversion; self->subbootversion = subbootversion; @@ -820,11 +805,9 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, self->loaded = TRUE; self->loaded_ts = stbuf.st_mtim; - ret = TRUE; if (out_changed) *out_changed = TRUE; - out: - return ret; + return TRUE; } int @@ -939,18 +922,13 @@ ostree_sysroot_get_repo (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - /* ostree_repo_open() is idempotent. */ if (!ostree_repo_open (self->repo, cancellable, error)) - goto out; + return FALSE; if (out_repo != NULL) *out_repo = g_object_ref (self->repo); - - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -1024,21 +1002,17 @@ parse_kernel_commandline (OstreeKernelArgs **out_args, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GFile) proc_cmdline = g_file_new_for_path ("/proc/cmdline"); g_autofree char *contents = NULL; gsize len; if (!g_file_load_contents (proc_cmdline, cancellable, &contents, &len, NULL, error)) - goto out; + return FALSE; g_strchomp (contents); - - ret = TRUE; *out_args = _ostree_kernel_args_from_string (contents); - out: - return ret; + return TRUE; } static gboolean @@ -1048,50 +1022,37 @@ find_booted_deployment (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; struct stat root_stbuf; struct stat self_stbuf; glnx_unref_object OstreeDeployment *ret_deployment = NULL; if (stat ("/", &root_stbuf) != 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "stat /"); if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; if (fstat (self->sysroot_fd, &self_stbuf) != 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fstat"); if (root_stbuf.st_dev == self_stbuf.st_dev && root_stbuf.st_ino == self_stbuf.st_ino) - { - guint i; - const char *bootlink_arg; + { __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kernel_args = NULL; - if (!parse_kernel_commandline (&kernel_args, cancellable, error)) - goto out; - - bootlink_arg = _ostree_kernel_args_get_last_value (kernel_args, "ostree"); + return FALSE; + + const char *bootlink_arg = _ostree_kernel_args_get_last_value (kernel_args, "ostree"); if (bootlink_arg) { - for (i = 0; i < deployments->len; i++) + for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); struct stat stbuf; if (fstatat (self->sysroot_fd, deployment_path, &stbuf, 0) != 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fstatat"); if (stbuf.st_dev == root_stbuf.st_dev && stbuf.st_ino == root_stbuf.st_ino) @@ -1102,11 +1063,7 @@ find_booted_deployment (OstreeSysroot *self, } if (ret_deployment == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unexpected state: ostree= kernel argument found, but / is not a deployment root"); - goto out; - } + return glnx_throw (error, "Unexpected state: ostree= kernel argument found, but / is not a deployment root"); } else { @@ -1114,10 +1071,8 @@ find_booted_deployment (OstreeSysroot *self, } } - ret = TRUE; ot_transfer_out_value (out_deployment, &ret_deployment); - out: - return ret; + return TRUE; } /** @@ -1157,7 +1112,7 @@ ostree_sysroot_get_merge_deployment (OstreeSysroot *self, if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) continue; - + return g_object_ref (deployment); } } @@ -1221,11 +1176,10 @@ ostree_sysroot_try_lock (OstreeSysroot *self, gboolean *out_acquired, GError **error) { - gboolean ret = FALSE; g_autoptr(GError) local_error = NULL; if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; /* Note use of LOCK_NB */ if (!glnx_make_lock_file (self->sysroot_fd, OSTREE_SYSROOT_LOCKFILE, @@ -1238,7 +1192,7 @@ ostree_sysroot_try_lock (OstreeSysroot *self, else { g_propagate_error (error, g_steal_pointer (&local_error)); - goto out; + return FALSE; } } else @@ -1246,9 +1200,7 @@ ostree_sysroot_try_lock (OstreeSysroot *self, *out_acquired = TRUE; } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -1279,7 +1231,7 @@ lock_in_thread (GTask *task, if (g_cancellable_set_error_if_cancelled (cancellable, &local_error)) ostree_sysroot_unlock (self); - + out: if (local_error) g_task_return_error (task, local_error); @@ -1340,77 +1292,49 @@ ostree_sysroot_init_osname (OstreeSysroot *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - const char *deploydir = glnx_strjoina ("ostree/deploy/", osname); - glnx_fd_close int dfd = -1; - if (!ensure_sysroot_fd (self, error)) - goto out; + return FALSE; + const char *deploydir = glnx_strjoina ("ostree/deploy/", osname); if (mkdirat (self->sysroot_fd, deploydir, 0777) < 0) - { - glnx_set_prefix_error_from_errno (error, "Creating %s", deploydir); - goto out; - } + return glnx_throw_errno_prefix (error, "Creating %s", deploydir); + glnx_fd_close int dfd = -1; if (!glnx_opendirat (self->sysroot_fd, deploydir, TRUE, &dfd, error)) - goto out; + return FALSE; if (mkdirat (dfd, "var", 0777) < 0) - { - glnx_set_prefix_error_from_errno (error, "Creating %s", "var"); - goto out; - } + return glnx_throw_errno_prefix (error, "Creating %s", "var"); /* This is a bit of a legacy hack...but we have to keep it around * now. We're ensuring core subdirectories of /var exist. */ if (mkdirat (dfd, "var/tmp", 0777) < 0) - { - glnx_set_prefix_error_from_errno (error, "Creating %s", "var/tmp"); - goto out; - } + return glnx_throw_errno_prefix (error, "Creating %s", "var/tmp"); if (fchmodat (dfd, "var/tmp", 01777, 0) < 0) - { - glnx_set_prefix_error_from_errno (error, "Fchmod %s", "var/tmp"); - goto out; - } + return glnx_throw_errno_prefix (error, "fchmod %s", "var/tmp"); if (mkdirat (dfd, "var/lib", 0777) < 0) - { - glnx_set_prefix_error_from_errno (error, "Creating %s", "var/tmp"); - goto out; - } + return glnx_throw_errno_prefix (error, "Creating %s", "var/tmp"); /* This needs to be available and properly labeled early during the boot * process (before tmpfiles.d kicks in), so that journald can flush logs from * the first boot there. https://bugzilla.redhat.com/show_bug.cgi?id=1265295 * */ if (mkdirat (dfd, "var/log", 0755) < 0) - { - glnx_set_prefix_error_from_errno (error, "Creating %s", "var/log"); - goto out; - } + return glnx_throw_errno_prefix (error, "Creating %s", "var/log"); if (symlinkat ("../run", dfd, "var/run") < 0) - { - glnx_set_prefix_error_from_errno (error, "Symlinking %s", "var/run"); - goto out; - } + return glnx_throw_errno_prefix (error, "Symlinking %s", "var/run"); if (symlinkat ("../run/lock", dfd, "var/lock") < 0) - { - glnx_set_prefix_error_from_errno (error, "Symlinking %s", "var/lock"); - goto out; - } + return glnx_throw_errno_prefix (error, "Symlinking %s", "var/lock"); if (!_ostree_sysroot_bump_mtime (self, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } /** From 63497c65f33c9e564f7c83b98b39a4fe7f588524 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 27 Apr 2017 14:24:20 -0400 Subject: [PATCH 71/94] checkout/commit: Use glnx_regfile_copy_bytes() if possible Rather than `g_output_stream_splice()`, where the input is a regular file. See https://github.com/GNOME/libglnx/pull/44 for some more information. I didn't try to measure the performance difference, but seeing the read()/write() to/from userspace mixed in with the pointless `poll()` annoyed me when reading strace. As a bonus, we will again start using reflinks (if available) for `/etc`, which is a regression from the https://github.com/ostreedev/ostree/pull/797 changes (which before used `glnx_file_copy_at()`). Also, for the first time we'll use reflinks when doing commits from file-backed content. This happens in `rpm-ostree compose tree` today for example. Update submodule: libglnx Closes: #817 Approved by: jlebon --- libglnx | 2 +- src/libostree/ostree-repo-checkout.c | 44 ++++++++++++-------- src/libostree/ostree-repo-commit.c | 61 +++++++++++++--------------- tests/ci-commitmessage-submodules.sh | 1 + 4 files changed, 58 insertions(+), 50 deletions(-) diff --git a/libglnx b/libglnx index 602fdd93..3a4d0f46 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 602fdd93cb7a339c6b5749eee73df926429a5ab8 +Subproject commit 3a4d0f4684f4653338c4756c8a1abfc6b5738467 diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 36be420b..50967769 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -102,7 +102,7 @@ fsync_is_enabled (OstreeRepo *self, static gboolean write_regular_file_content (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options, - GOutputStream *output, + int outfd, GFileInfo *file_info, GVariant *xattrs, GInputStream *input, @@ -110,40 +110,54 @@ write_regular_file_content (OstreeRepo *self, GError **error) { const OstreeRepoCheckoutMode mode = options->mode; + g_autoptr(GOutputStream) outstream = NULL; - if (g_output_stream_splice (output, input, 0, - cancellable, error) < 0) - return FALSE; + if (G_IS_FILE_DESCRIPTOR_BASED (input)) + { + int infd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*) input); + guint64 len = g_file_info_get_size (file_info); - if (!g_output_stream_flush (output, cancellable, error)) - return FALSE; + if (glnx_regfile_copy_bytes (infd, outfd, (off_t)len, TRUE) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); + } + else + { + outstream = g_unix_output_stream_new (outfd, FALSE); + if (g_output_stream_splice (outstream, input, 0, + cancellable, error) < 0) + return FALSE; - int fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)output); + if (!g_output_stream_flush (outstream, cancellable, error)) + return FALSE; + } if (mode != OSTREE_REPO_CHECKOUT_MODE_USER) { - if (TEMP_FAILURE_RETRY (fchown (fd, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), + if (TEMP_FAILURE_RETRY (fchown (outfd, g_file_info_get_attribute_uint32 (file_info, "unix::uid"), g_file_info_get_attribute_uint32 (file_info, "unix::gid"))) < 0) return glnx_throw_errno (error); - if (TEMP_FAILURE_RETRY (fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"))) < 0) + if (TEMP_FAILURE_RETRY (fchmod (outfd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"))) < 0) return glnx_throw_errno (error); if (xattrs) { - if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error)) + if (!glnx_fd_set_all_xattrs (outfd, xattrs, cancellable, error)) return FALSE; } } if (fsync_is_enabled (self, options)) { - if (fsync (fd) == -1) + if (fsync (outfd) == -1) return glnx_throw_errno (error); } - if (!g_output_stream_close (output, cancellable, error)) - return FALSE; + if (outstream) + { + if (!g_output_stream_close (outstream, cancellable, error)) + return FALSE; + } return TRUE; } @@ -231,7 +245,6 @@ create_file_copy_from_input_at (OstreeRepo *repo, else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) { glnx_fd_close int temp_fd = -1; - g_autoptr(GOutputStream) temp_out = NULL; guint32 file_mode; GLnxLinkTmpfileReplaceMode replace_mode; @@ -244,7 +257,6 @@ create_file_copy_from_input_at (OstreeRepo *repo, &temp_fd, &temp_filename, error)) return FALSE; - temp_out = g_unix_output_stream_new (temp_fd, FALSE); if (sepolicy_enabled) { @@ -258,7 +270,7 @@ create_file_copy_from_input_at (OstreeRepo *repo, return glnx_throw_errno_prefix (error, "Setting security.selinux"); } - if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input, + if (!write_regular_file_content (repo, options, temp_fd, file_info, xattrs, input, cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 664016b8..2e9f9243 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -458,29 +458,19 @@ add_size_index_to_metadata (OstreeRepo *self, } static gboolean -fallocate_stream (GFileDescriptorBased *stream, - goffset size, - GCancellable *cancellable, - GError **error) +ot_fallocate (int fd, goffset size, GError **error) { - gboolean ret = FALSE; - int fd = g_file_descriptor_based_get_fd (stream); + if (size == 0) + return TRUE; - if (size > 0) + int r = posix_fallocate (fd, 0, size); + if (r != 0) { - int r = posix_fallocate (fd, 0, size); - if (r != 0) - { - /* posix_fallocate is a weird deviation from errno standards */ - errno = r; - glnx_set_error_from_errno (error); - goto out; - } + /* posix_fallocate is a weird deviation from errno standards */ + errno = r; + return glnx_throw_errno_prefix (error, "fallocate"); } - - ret = TRUE; - out: - return ret; + return TRUE; } gboolean @@ -510,11 +500,10 @@ _ostree_repo_open_content_bare (OstreeRepo *self, &fd, &temp_filename, error)) goto out; - ret_stream = g_unix_output_stream_new (fd, TRUE); - - if (!fallocate_stream ((GFileDescriptorBased*)ret_stream, content_len, - cancellable, error)) + if (!ot_fallocate (fd, content_len, error)) goto out; + + ret_stream = g_unix_output_stream_new (fd, TRUE); } ret = TRUE; @@ -569,7 +558,6 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GOutputStream) temp_out = NULL; glnx_fd_close int temp_fd = -1; g_autofree char *temp_filename = NULL; @@ -577,20 +565,27 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, &temp_fd, &temp_filename, error)) return FALSE; - temp_out = g_unix_output_stream_new (temp_fd, FALSE); - if (!fallocate_stream ((GFileDescriptorBased*)temp_out, length, - cancellable, error)) + if (!ot_fallocate (temp_fd, length, error)) return FALSE; - if (g_output_stream_splice (temp_out, input, 0, - cancellable, error) < 0) - return FALSE; - if (fchmod (temp_fd, 0644) < 0) + if (G_IS_FILE_DESCRIPTOR_BASED (input)) { - glnx_set_error_from_errno (error); - return FALSE; + int infd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*) input); + if (glnx_regfile_copy_bytes (infd, temp_fd, (off_t)length, TRUE) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); } + else + { + g_autoptr(GOutputStream) temp_out = g_unix_output_stream_new (temp_fd, FALSE); + if (g_output_stream_splice (temp_out, input, 0, + cancellable, error) < 0) + return FALSE; + } + + if (fchmod (temp_fd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + *out_fd = temp_fd; temp_fd = -1; *out_path = g_steal_pointer (&temp_filename); return TRUE; diff --git a/tests/ci-commitmessage-submodules.sh b/tests/ci-commitmessage-submodules.sh index 77a4e1a1..2403579d 100755 --- a/tests/ci-commitmessage-submodules.sh +++ b/tests/ci-commitmessage-submodules.sh @@ -40,6 +40,7 @@ git log --pretty=oneline origin/master.. | while read logline; do echo "Commit $commit modifies submodule: $submodule" expected_match="Update submodule: $submodule" if ! grep -q -e "$expected_match" ${tmpd}/log.txt; then + sed -e 's,^,# ,' < ${tmpd}/log.txt echo "error: Commit message for ${commit} changes a submodule, but does not match regex ${expected_match}" exit 1 fi From 986e05e3fd59abc6f837cc9a6282e75b9596ad76 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 May 2017 11:08:12 -0400 Subject: [PATCH 72/94] lib/prune: Complete porting to new code style Only non-mechanical bit here was creating a local autoptr for a bit where we'd previously done an unref for a struct member. Closes: #847 Approved by: jlebon --- src/libostree/ostree-repo-prune.c | 90 +++++++++++-------------------- 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 1b2a8257..76d64794 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -42,21 +42,14 @@ prune_commitpartial_file (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *path = _ostree_get_commitpartial_path (checksum); - if (unlinkat (repo->repo_dir_fd, path, 0) != 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "unlinkat"); } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -67,7 +60,6 @@ maybe_prune_loose_object (OtPruneData *data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) key = NULL; key = ostree_object_name_serialize (checksum, objtype); @@ -83,16 +75,16 @@ maybe_prune_loose_object (OtPruneData *data, if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!prune_commitpartial_file (data->repo, checksum, cancellable, error)) - goto out; + return FALSE; } if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, &storage_size, cancellable, error)) - goto out; + return FALSE; if (!ostree_repo_delete_object (data->repo, objtype, checksum, cancellable, error)) - goto out; + return FALSE; data->freed_bytes += storage_size; } @@ -111,9 +103,7 @@ maybe_prune_loose_object (OtPruneData *data, data->n_reachable_content++; } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -121,25 +111,22 @@ _ostree_repo_prune_tmp (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - glnx_fd_close int fd = -1; - if (self->cache_dir_fd == -1) return TRUE; - fd = glnx_opendirat_with_errno (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, FALSE); + glnx_fd_close int fd = glnx_opendirat_with_errno (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, FALSE); if (fd < 0) { + /* Note early return */ if (errno == ENOENT) - ret = TRUE; + return TRUE; else - glnx_set_error_from_errno (error); - goto out; + return glnx_throw_errno_prefix (error, "opendirat(%s)", _OSTREE_SUMMARY_CACHE_DIR); } + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_take_fd (dup (fd), &dfd_iter, error)) - goto out; + return FALSE; while (TRUE) { @@ -148,8 +135,7 @@ _ostree_repo_prune_tmp (OstreeRepo *self, struct dirent *dent; if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error)) - goto out; - + return FALSE; if (dent == NULL) break; @@ -167,17 +153,11 @@ _ostree_repo_prune_tmp (OstreeRepo *self, dent->d_name[len - 4] = '.'; if (unlinkat (fd, dent->d_name, 0) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "unlinkat"); } } - ret = TRUE; - - out: - return ret; + return TRUE; } @@ -198,22 +178,17 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GPtrArray) deltas = NULL; - guint i; - if (!ostree_repo_list_static_delta_names (self, &deltas, cancellable, error)) - goto out; + return FALSE; - for (i = 0; i < deltas->len; i++) + for (guint i = 0; i < deltas->len; i++) { const char *deltaname = deltas->pdata[i]; const char *dash = strchr (deltaname, '-'); const char *to = NULL; - gboolean have_commit; g_autofree char *from = NULL; - g_autofree char *deltadir = NULL; if (!dash) { @@ -232,26 +207,24 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, } else { + gboolean have_commit; if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, to, &have_commit, cancellable, error)) - goto out; + return FALSE; if (have_commit) continue; } g_debug ("Trying to prune static delta %s", deltaname); - deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); - + g_autofree char *deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); if (!glnx_shutil_rm_rf_at (self->repo_dir_fd, deltadir, - cancellable, error)) - goto out; + cancellable, error)) + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -264,13 +237,14 @@ repo_prune_internal (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; GHashTableIter hash_iter; gpointer key, value; OtPruneData data = { 0, }; data.repo = self; - data.reachable = g_hash_table_ref (options->reachable); + /* We unref this when we're done */ + g_autoptr(GHashTable) reachable_owned = g_hash_table_ref (options->reachable); + data.reachable = reachable_owned; g_hash_table_iter_init (&hash_iter, objects); while (g_hash_table_iter_next (&hash_iter, &key, &value)) @@ -289,24 +263,20 @@ repo_prune_internal (OstreeRepo *self, if (!maybe_prune_loose_object (&data, options->flags, checksum, objtype, cancellable, error)) - goto out; + return FALSE; } if (!ostree_repo_prune_static_deltas (self, NULL, cancellable, error)) - goto out; + return FALSE; if (!_ostree_repo_prune_tmp (self, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; *out_objects_total = (data.n_reachable_meta + data.n_unreachable_meta + data.n_reachable_content + data.n_unreachable_content); *out_objects_pruned = (data.n_unreachable_meta + data.n_unreachable_content); *out_pruned_object_size_total = data.freed_bytes; - out: - if (data.reachable) - g_hash_table_unref (data.reachable); - return ret; + return TRUE; } /** From 7896bcbe6595e4dd076a06a024d058193fa0f09c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 May 2017 21:43:26 -0400 Subject: [PATCH 73/94] lib/checkout: Move special case for subpath of file to toplevel Since we now have a cleaner separation of "toplevel checkout prep" versus "recursive checkout", handle the special case of checking out a single file at first rather than later. Prep for future work in optimizing this function more. Closes: #848 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 50967769..9c1420ee 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -666,15 +666,6 @@ checkout_tree_at_recurse (OstreeRepo *self, return FALSE; } - /* Note early return here! */ - if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) - return checkout_one_file_at (self, options, state, - (GFile *) source, - source_info, - destination_dfd, - g_file_info_get_name (source_info), - cancellable, error); - g_autoptr(GFileEnumerator) dir_enum = g_file_enumerate_children ((GFile*)source, OSTREE_GIO_FAST_QUERYINFO, @@ -789,6 +780,15 @@ checkout_tree_at (OstreeRepo *self, g_assert (options->force_copy); } + /* Special case handling for subpath of a non-directory */ + if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) + return checkout_one_file_at (self, options, &state, + (GFile *) source, + source_info, + destination_parent_fd, + g_file_info_get_name (source_info), + cancellable, error); + /* Cache any directory metadata we read during this operation; * see commit b7afe91e21143d7abb0adde440683a52712aa246 */ From e6f17b949d8214049faca04d01d82bb28703cc77 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 May 2017 21:40:50 -0400 Subject: [PATCH 74/94] lib/checkout: Optimize checkout by avoiding OstreeRepoFile recusion Looking at `perf record ostree checkout`, some things stand out; e.g.: ``` + 27.63% 0.07% ostree libgio-2.0.so.0.5000.3 [.] g_file_enumerator_iterate + 22.74% 0.28% ostree libostree-1.so.1.0.0 [.] ostree_repo_file_tree_query_child + 13.74% 0.08% ostree libostree-1.so.1.0.0 [.] ot_variant_bsearch_str ``` The GIO abstractions are already fairly heavyweight, and `OstreeRepoFile` mallocs a lot too. Make things more efficient here by dropping the GIO bits for reading ostree data - we just read from the variants directly and iterate over them. The end result here is that according to perf we go from ~40% of our time in the kernel to ~70%, and things like `g_file_enumerator_iterate()` drop entirely out of the hot set. Closes: #848 Approved by: jlebon --- src/libostree/ostree-core-private.h | 8 ++ src/libostree/ostree-repo-checkout.c | 153 ++++++++++++++++----------- 2 files changed, 97 insertions(+), 64 deletions(-) diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a66a068f..72b88aba 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -90,6 +90,14 @@ _ostree_make_temporary_symlink_at (int tmp_dirfd, GFileInfo * _ostree_header_gfile_info_new (mode_t mode, uid_t uid, gid_t gid); +static inline void +_ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) +{ + const guint8*csum = ostree_checksum_bytes_peek (csum_v); + g_assert (csum); + ostree_checksum_inplace_from_bytes (csum, buf); +} + /* XX/checksum-2.extension, but let's just use 256 for a * bit of overkill. */ diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 9c1420ee..b4298ba4 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -361,8 +361,7 @@ static gboolean checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, CheckoutState *state, - GFile *source, - GFileInfo *source_info, + const char *checksum, int destination_dfd, const char *destination_name, GCancellable *cancellable, @@ -371,8 +370,14 @@ checkout_one_file_at (OstreeRepo *repo, gboolean need_copy = TRUE; gboolean is_bare_user_symlink = FALSE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; + + /* FIXME - avoid the GFileInfo here */ + g_autoptr(GFileInfo) source_info = NULL; + if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, NULL, + cancellable, error)) + return FALSE; + const gboolean is_symlink = (g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK); - const char *checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source); const gboolean is_whiteout = (!is_symlink && options->process_whiteouts && g_str_has_prefix (destination_name, WHITEOUT_PREFIX)); @@ -592,21 +597,33 @@ checkout_tree_at_recurse (OstreeRepo *self, CheckoutState *state, int destination_parent_fd, const char *destination_name, - OstreeRepoFile *source, - GFileInfo *source_info, + const char *dirtree_checksum, + const char *dirmeta_checksum, GCancellable *cancellable, GError **error) { gboolean did_exist = FALSE; const gboolean sepolicy_enabled = options->sepolicy && !self->disable_xattrs; + g_autoptr(GVariant) dirtree = NULL; + g_autoptr(GVariant) dirmeta = NULL; g_autoptr(GVariant) xattrs = NULL; g_autoptr(GVariant) modified_xattrs = NULL; - if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) - { - if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error)) - return FALSE; - } + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_DIR_TREE, + dirtree_checksum, &dirtree, error)) + return FALSE; + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_DIR_META, + dirmeta_checksum, &dirmeta, error)) + return FALSE; + + /* Parse OSTREE_OBJECT_TYPE_DIR_META */ + guint32 uid, gid, mode; + g_variant_get (dirmeta, "(uuu@a(ayay))", + &uid, &gid, &mode, + options->mode != OSTREE_REPO_CHECKOUT_MODE_USER ? &xattrs : NULL); + uid = GUINT32_FROM_BE (uid); + gid = GUINT32_FROM_BE (gid); + mode = GUINT32_FROM_BE (mode); /* First, make the directory. Push a new scope in case we end up using * setfscreatecon(). @@ -623,8 +640,7 @@ checkout_tree_at_recurse (OstreeRepo *self, if (!_ostree_sepolicy_preparefscreatecon (&fscreatecon, options->sepolicy, state->selabel_path_buf->str, - g_file_info_get_attribute_uint32 (source_info, "unix::mode"), - error)) + mode, error)) return FALSE; } @@ -666,72 +682,78 @@ checkout_tree_at_recurse (OstreeRepo *self, return FALSE; } - g_autoptr(GFileEnumerator) dir_enum = - g_file_enumerate_children ((GFile*)source, - OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, - error); - if (!dir_enum) - return FALSE; + GString *selabel_path_buf = state->selabel_path_buf; + /* Process files in this subdir */ + { g_autoptr(GVariant) dir_file_contents = g_variant_get_child_value (dirtree, 0); + GVariantIter viter; + g_variant_iter_init (&viter, dir_file_contents); + const char *fname; + g_autoptr(GVariant) contents_csum_v = NULL; + while (g_variant_iter_loop (&viter, "(&s@ay)", &fname, &contents_csum_v)) + { + const size_t namelen = strlen (fname); + if (selabel_path_buf) + g_string_append_len (selabel_path_buf, fname, namelen); - while (TRUE) - { - GFileInfo *file_info; - GFile *src_child; - const char *name; - GString *selabel_path_buf = state->selabel_path_buf; + char tmp_checksum[OSTREE_SHA256_STRING_LEN+1]; + _ostree_checksum_inplace_from_bytes_v (contents_csum_v, tmp_checksum); - if (!g_file_enumerator_iterate (dir_enum, &file_info, &src_child, - cancellable, error)) - return FALSE; - if (file_info == NULL) - break; + if (!checkout_one_file_at (self, options, state, + tmp_checksum, + destination_dfd, fname, + cancellable, error)) + return FALSE; - name = g_file_info_get_name (file_info); - size_t namelen = strlen (name); - if (selabel_path_buf) - g_string_append_len (selabel_path_buf, name, namelen); + if (selabel_path_buf) + g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); + } + contents_csum_v = NULL; /* iter_loop freed it */ + } - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) - { - if (selabel_path_buf) + /* Process subdirectories */ + { g_autoptr(GVariant) dir_subdirs = g_variant_get_child_value (dirtree, 1); + const char *dname; + g_autoptr(GVariant) subdirtree_csum_v = NULL; + g_autoptr(GVariant) subdirmeta_csum_v = NULL; + GVariantIter viter; + g_variant_iter_init (&viter, dir_subdirs); + while (g_variant_iter_loop (&viter, "(&s@ay@ay)", &dname, + &subdirtree_csum_v, &subdirmeta_csum_v)) + { + const size_t namelen = strlen (dname); + if (selabel_path_buf) + { + g_string_append_len (selabel_path_buf, dname, namelen); g_string_append_c (selabel_path_buf, '/'); - if (!checkout_tree_at_recurse (self, options, state, - destination_dfd, name, - (OstreeRepoFile*)src_child, file_info, - cancellable, error)) - return FALSE; - if (state->selabel_path_buf) - g_string_truncate (selabel_path_buf, state->selabel_path_buf->len-1); - } - else - { - if (!checkout_one_file_at (self, options, state, - src_child, file_info, - destination_dfd, name, - cancellable, error)) - return FALSE; - } + } - if (selabel_path_buf) - g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); - } + char subdirtree_checksum[OSTREE_SHA256_STRING_LEN+1]; + _ostree_checksum_inplace_from_bytes_v (subdirtree_csum_v, subdirtree_checksum); + char subdirmeta_checksum[OSTREE_SHA256_STRING_LEN+1]; + _ostree_checksum_inplace_from_bytes_v (subdirmeta_csum_v, subdirmeta_checksum); + if (!checkout_tree_at_recurse (self, options, state, + destination_dfd, dname, + subdirtree_checksum, subdirmeta_checksum, + cancellable, error)) + return FALSE; + + if (selabel_path_buf) + g_string_truncate (selabel_path_buf, selabel_path_buf->len - namelen); + } + } /* We do fchmod/fchown last so that no one else could access the * partially created directory and change content we're laying out. */ if (!did_exist) { - if (TEMP_FAILURE_RETRY (fchmod (destination_dfd, g_file_info_get_attribute_uint32 (source_info, "unix::mode"))) < 0) + if (TEMP_FAILURE_RETRY (fchmod (destination_dfd, mode)) < 0) return glnx_throw_errno (error); } if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) { - if (TEMP_FAILURE_RETRY (fchown (destination_dfd, - g_file_info_get_attribute_uint32 (source_info, "unix::uid"), - g_file_info_get_attribute_uint32 (source_info, "unix::gid"))) < 0) + if (TEMP_FAILURE_RETRY (fchown (destination_dfd, uid, gid)) < 0) return glnx_throw_errno (error); } @@ -783,8 +805,7 @@ checkout_tree_at (OstreeRepo *self, /* Special case handling for subpath of a non-directory */ if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) return checkout_one_file_at (self, options, &state, - (GFile *) source, - source_info, + ostree_repo_file_get_checksum (source), destination_parent_fd, g_file_info_get_name (source_info), cancellable, error); @@ -794,9 +815,13 @@ checkout_tree_at (OstreeRepo *self, */ g_auto(OstreeRepoMemoryCacheRef) memcache_ref; _ostree_repo_memory_cache_ref_init (&memcache_ref, self); + + g_assert_cmpint (g_file_info_get_file_type (source_info), ==, G_FILE_TYPE_DIRECTORY); + const char *dirtree_checksum = ostree_repo_file_tree_get_contents_checksum (source); + const char *dirmeta_checksum = ostree_repo_file_tree_get_metadata_checksum (source); return checkout_tree_at_recurse (self, options, &state, destination_parent_fd, destination_name, - source, source_info, + dirtree_checksum, dirmeta_checksum, cancellable, error); } From 964ca9d4341fe565bf3ebf97d0f3af84e5ff4346 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 May 2017 22:21:27 -0400 Subject: [PATCH 75/94] repo: Fix double close() in summary generation Happened to notice this while doing a style port. Closes: #849 Approved by: jlebon --- src/libostree/ostree-repo.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 76ae0d9b..1f02007f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4582,6 +4582,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, in_stream = g_unix_input_stream_new (superblock_file_fd, TRUE); if (!in_stream) goto out; + superblock_file_fd = -1; /* Transfer ownership */ if (!ot_gio_checksum_stream (in_stream, &csum, From 01772149827d3ea7c86dc2bc3110768834d7ac3f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 10 May 2017 22:19:14 -0400 Subject: [PATCH 76/94] lib/repo: Port more of GPG and summary functions to new code style These ones were pretty easy, not sure why I didn't do them in an earlier pass. Closes: #849 Approved by: jlebon --- src/libostree/ostree-repo.c | 161 ++++++++++++++---------------------- 1 file changed, 60 insertions(+), 101 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 1f02007f..93e1888e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3911,77 +3911,64 @@ ostree_repo_sign_commit (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GBytes) commit_data = NULL; g_autoptr(GBytes) signature = NULL; - g_autoptr(GVariant) commit_variant = NULL; - g_autoptr(GVariant) old_metadata = NULL; - g_autoptr(GVariant) new_metadata = NULL; - glnx_unref_object OstreeGpgVerifyResult *result = NULL; - GError *local_error = NULL; + g_autoptr(GVariant) commit_variant = NULL; if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit_checksum, &commit_variant, error)) - { - g_prefix_error (error, "Failed to read commit: "); - goto out; - } + return g_prefix_error (error, "Failed to read commit: "), FALSE; + g_autoptr(GVariant) old_metadata = NULL; if (!ostree_repo_read_commit_detached_metadata (self, commit_checksum, &old_metadata, cancellable, error)) - { - g_prefix_error (error, "Failed to read detached metadata: "); - goto out; - } + return g_prefix_error (error, "Failed to read detached metadata: "), FALSE; commit_data = g_variant_get_data_as_bytes (commit_variant); /* The verify operation is merely to parse any existing signatures to * check if the commit has already been signed with the given key ID. * We want to avoid storing duplicate signatures in the metadata. */ - result = _ostree_repo_gpg_verify_with_metadata (self, - commit_data, - old_metadata, - NULL, NULL, NULL, - cancellable, - &local_error); - - /* "Not found" just means the commit is not yet signed. That's okay. */ - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + g_autoptr(GError) local_error = NULL; + glnx_unref_object OstreeGpgVerifyResult *result + =_ostree_repo_gpg_verify_with_metadata (self, commit_data, old_metadata, + NULL, NULL, NULL, + cancellable, &local_error); + if (!result) { - g_clear_error (&local_error); - } - else if (local_error != NULL) - { - g_propagate_error (error, local_error); - goto out; + /* "Not found" just means the commit is not yet signed. That's okay. */ + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + { + g_clear_error (&local_error); + } + else + return g_propagate_error (error, g_steal_pointer (&local_error)), FALSE; } else if (ostree_gpg_verify_result_lookup (result, key_id, NULL)) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_EXISTS, "Commit is already signed with GPG key %s", key_id); - goto out; + return FALSE; } if (!sign_data (self, commit_data, key_id, homedir, &signature, cancellable, error)) - goto out; + return FALSE; - new_metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature); + g_autoptr(GVariant) new_metadata = + _ostree_detached_metadata_append_gpg_sig (old_metadata, signature); if (!ostree_repo_write_commit_detached_metadata (self, commit_checksum, new_metadata, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; -out: - return ret; + return TRUE; } /** @@ -4005,8 +3992,9 @@ ostree_repo_sign_delta (OstreeRepo *self, const gchar *homedir, GCancellable *cancellable, GError **error) -{ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "ostree_repo_sign_delta is deprecated"); +{ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "ostree_repo_sign_delta is deprecated"); return FALSE; } @@ -4027,34 +4015,29 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GBytes) summary_data = NULL; - g_autoptr(GVariant) existing_signatures = NULL; - g_autoptr(GVariant) new_metadata = NULL; - g_autoptr(GVariant) normalized = NULL; - guint i; - - summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error); + g_autoptr(GBytes) summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error); if (!summary_data) - goto out; + return FALSE; + g_autoptr(GVariant) existing_signatures = NULL; if (!ot_util_variant_map_at (self->repo_dir_fd, "summary.sig", G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), OT_VARIANT_MAP_ALLOW_NOENT, &existing_signatures, error)) - goto out; + return FALSE; - for (i = 0; key_id[i]; i++) + g_autoptr(GVariant) new_metadata = NULL; + for (guint i = 0; key_id[i]; i++) { g_autoptr(GBytes) signature_data = NULL; if (!sign_data (self, summary_data, key_id[i], homedir, &signature_data, cancellable, error)) - goto out; + return FALSE; new_metadata = _ostree_detached_metadata_append_gpg_sig (existing_signatures, signature_data); } - normalized = g_variant_get_normal_form (new_metadata); + g_autoptr(GVariant) normalized = g_variant_get_normal_form (new_metadata); if (!_ostree_repo_file_replace_contents (self, self->repo_dir_fd, @@ -4062,11 +4045,9 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, g_variant_get_data (normalized), g_variant_get_size (normalized), cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } /* Special remote for _ostree_repo_gpg_verify_with_metadata() */ @@ -4493,35 +4474,26 @@ ostree_repo_regenerate_summary (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GHashTable) refs = NULL; - g_autoptr(GVariantBuilder) refs_builder = NULL; - g_autoptr(GVariant) summary = NULL; - GList *ordered_keys = NULL; - GList *iter = NULL; - g_auto(GVariantDict) additional_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; - if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) - goto out; + return FALSE; + g_auto(GVariantDict) additional_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; g_variant_dict_init (&additional_metadata_builder, additional_metadata); - refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); + g_autoptr(GVariantBuilder) refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); - ordered_keys = g_hash_table_get_keys (refs); + g_autoptr(GList) ordered_keys = g_hash_table_get_keys (refs); ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); - for (iter = ordered_keys; iter; iter = iter->next) + for (GList *iter = ordered_keys; iter; iter = iter->next) { const char *ref = iter->data; const char *commit = g_hash_table_lookup (refs, ref); - g_autofree char *remotename = NULL; - g_autoptr(GVariant) commit_obj = NULL; g_auto(GVariantDict) commit_metadata_builder = OT_VARIANT_BUILDER_INITIALIZER; - guint64 commit_timestamp; - g_autoptr(GDateTime) dt = NULL; g_assert (commit); + g_autofree char *remotename = NULL; if (!ostree_parse_refspec (ref, &remotename, NULL, NULL)) g_assert_not_reached (); @@ -4529,14 +4501,15 @@ ostree_repo_regenerate_summary (OstreeRepo *self, if (remotename != NULL) continue; + g_autoptr(GVariant) commit_obj = NULL; if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) - goto out; + return FALSE; g_variant_dict_init (&commit_metadata_builder, NULL); /* Forward the commit’s timestamp if it’s valid. */ - commit_timestamp = ostree_commit_get_timestamp (commit_obj); - dt = g_date_time_new_from_unix_utc (commit_timestamp); + guint64 commit_timestamp = ostree_commit_get_timestamp (commit_obj); + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (commit_timestamp); if (dt != NULL) g_variant_dict_insert_value (&commit_metadata_builder, OSTREE_COMMIT_TIMESTAMP, @@ -4551,44 +4524,36 @@ ostree_repo_regenerate_summary (OstreeRepo *self, { - guint i; g_autoptr(GPtrArray) delta_names = NULL; g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) - goto out; + return FALSE; g_variant_dict_init (&deltas_builder, NULL); - for (i = 0; i < delta_names->len; i++) + for (guint i = 0; i < delta_names->len; i++) { g_autofree char *from = NULL; g_autofree char *to = NULL; - g_autofree guchar *csum = NULL; - g_autofree char *superblock = NULL; - glnx_fd_close int superblock_file_fd = -1; - g_autoptr(GInputStream) in_stream = NULL; - if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) - goto out; + return FALSE; - superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); - superblock_file_fd = openat (self->repo_dir_fd, superblock, O_RDONLY | O_CLOEXEC); + g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to); + glnx_fd_close int superblock_file_fd = openat (self->repo_dir_fd, superblock, O_RDONLY | O_CLOEXEC); if (superblock_file_fd == -1) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno (error); - in_stream = g_unix_input_stream_new (superblock_file_fd, TRUE); + g_autoptr(GInputStream) in_stream = g_unix_input_stream_new (superblock_file_fd, FALSE); if (!in_stream) - goto out; + return FALSE; superblock_file_fd = -1; /* Transfer ownership */ + g_autofree guchar *csum = NULL; if (!ot_gio_checksum_stream (in_stream, &csum, cancellable, error)) - goto out; + return FALSE; g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], ot_gvariant_new_bytearray (csum, 32)); } @@ -4601,6 +4566,7 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_new_uint64 (GUINT64_TO_BE (g_get_real_time () / G_USEC_PER_SEC))); } + g_autoptr(GVariant) summary = NULL; { g_autoptr(GVariantBuilder) summary_builder = g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT); @@ -4618,22 +4584,15 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_get_size (summary), cancellable, error)) - goto out; + return FALSE; if (unlinkat (self->repo_dir_fd, "summary.sig", 0) < 0) { if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "unlinkat"); } - ret = TRUE; - out: - if (ordered_keys) - g_list_free (ordered_keys); - return ret; + return TRUE; } gboolean From ce4d21bc1721634d6353dec63f932d5ee417d853 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 May 2017 09:08:20 -0400 Subject: [PATCH 77/94] checkout: Plug a memleak of the state stringbuf A struct without a cleanup macro is a struct likely to leak. Closes: #850 Approved by: jlebon --- src/libostree/ostree-repo-checkout.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index b4298ba4..f5ef9517 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -40,6 +40,14 @@ typedef struct { GString *selabel_path_buf; } CheckoutState; +static void +checkout_state_clear (CheckoutState *state) +{ + if (state->selabel_path_buf) + g_string_free (state->selabel_path_buf, TRUE); +} +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(CheckoutState, checkout_state_clear); + static gboolean checkout_object_for_uncompressed_cache (OstreeRepo *self, const char *loose_path, @@ -787,7 +795,7 @@ checkout_tree_at (OstreeRepo *self, GCancellable *cancellable, GError **error) { - CheckoutState state = { 0, }; + g_auto(CheckoutState) state = { 0, }; // If SELinux labeling is enabled, we need to keep track of the full path string if (options->sepolicy) { From b83d509e78ff37edf384a9417d0d70af5982a31c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 May 2017 10:59:21 -0400 Subject: [PATCH 78/94] =?UTF-8?q?tree-wide:=20Switch=20tabs=20=E2=AD=BE=20?= =?UTF-8?q?in=20various=20files=20over=20to=20spaces=20=E2=90=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As $DEITY intended. I was reading the `prepare-root.c` code and the indentation damage was distracting. Squash tabs that have leaked into various places in the code. I didn't yet touch the `src/libostree` bits as that has higher potential for conflict. Closes: #852 Approved by: jlebon --- src/ostree/ostree-trivial-httpd.c | 8 +-- src/rofiles-fuse/main.c | 79 ++++++++++++++-------------- src/switchroot/ostree-prepare-root.c | 34 ++++++------ src/switchroot/ostree-remount.c | 4 +- tests/readdir-rand.c | 60 ++++++++++----------- tests/test-rollsum.c | 2 +- 6 files changed, 93 insertions(+), 94 deletions(-) diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index d6f0c4d4..176f5ec7 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -457,10 +457,10 @@ httpd_callback (SoupServer *server, SoupMessage *msg, static void on_dir_changed (GFileMonitor *mon, - GFile *file, - GFile *other, - GFileMonitorEvent event, - gpointer user_data) + GFile *file, + GFile *other, + GFileMonitorEvent event, + gpointer user_data) { OtTrivialHttpd *self = user_data; diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c index 3f0832f5..12a9d886 100644 --- a/src/rofiles-fuse/main.c +++ b/src/rofiles-fuse/main.c @@ -56,12 +56,12 @@ callback_getattr (const char *path, struct stat *st_data) if (!*path) { if (fstat (basefd, st_data) == -1) - return -errno; + return -errno; } else { if (fstatat (basefd, path, st_data, AT_SYMLINK_NOFOLLOW) == -1) - return -errno; + return -errno; } return 0; } @@ -85,7 +85,7 @@ callback_readlink (const char *path, char *buf, size_t size) static int callback_readdir (const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi) + off_t offset, struct fuse_file_info *fi) { DIR *dp; struct dirent *de; @@ -102,7 +102,7 @@ callback_readdir (const char *path, void *buf, fuse_fill_dir_t filler, { dfd = openat (basefd, path, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); if (dfd == -1) - return -errno; + return -errno; } /* Transfers ownership of fd */ @@ -117,7 +117,7 @@ callback_readdir (const char *path, void *buf, fuse_fill_dir_t filler, st.st_ino = de->d_ino; st.st_mode = de->d_type << 12; if (filler (buf, de->d_name, &st, 0)) - break; + break; } (void) closedir (dp); @@ -170,7 +170,7 @@ callback_symlink (const char *from, const char *to) if (fstatat (basefd, to, &stbuf, AT_SYMLINK_NOFOLLOW) == -1) { fprintf (stderr, "Failed to find newly created symlink '%s': %s\n", - to, g_strerror (errno)); + to, g_strerror (errno)); exit (EXIT_FAILURE); } return 0; @@ -209,19 +209,19 @@ can_write (const char *path) if (fstatat (basefd, path, &stbuf, 0) == -1) { if (errno == ENOENT) - return 0; + return 0; else - return -errno; + return -errno; } if (stbuf_is_regfile_hardlinked (&stbuf)) return -EROFS; return 0; } -#define VERIFY_WRITE(path) do { \ - int r = can_write (path); \ - if (r != 0) \ - return r; \ +#define VERIFY_WRITE(path) do { \ + int r = can_write (path); \ + if (r != 0) \ + return r; \ } while (0) static int @@ -366,7 +366,7 @@ callback_read_buf (const char *path, struct fuse_bufvec **bufp, static int callback_read (const char *path, char *buf, size_t size, off_t offset, - struct fuse_file_info *finfo) + struct fuse_file_info *finfo) { int r; r = pread (finfo->fh, buf, size, offset); @@ -390,7 +390,7 @@ callback_write_buf (const char *path, struct fuse_bufvec *buf, off_t offset, static int callback_write (const char *path, const char *buf, size_t size, off_t offset, - struct fuse_file_info *finfo) + struct fuse_file_info *finfo) { int r; r = pwrite (finfo->fh, buf, size, offset); @@ -426,7 +426,7 @@ static int callback_access (const char *path, int mode) { path = ENSURE_RELPATH (path); - + /* Apparently at least GNU coreutils rm calls `faccessat(W_OK)` * before trying to do an unlink. So...we'll just lie about * writable access here. @@ -438,14 +438,14 @@ callback_access (const char *path, int mode) static int callback_setxattr (const char *path, const char *name, const char *value, - size_t size, int flags) + size_t size, int flags) { return -ENOTSUP; } static int callback_getxattr (const char *path, const char *name, char *value, - size_t size) + size_t size) { return -ENOTSUP; } @@ -503,8 +503,7 @@ struct fuse_operations callback_oper = { .removexattr = callback_removexattr }; -enum -{ +enum { KEY_HELP, KEY_VERSION, }; @@ -513,19 +512,19 @@ static void usage (const char *progname) { fprintf (stdout, - "usage: %s basepath mountpoint [options]\n" - "\n" - " Makes basepath visible at mountpoint such that files are read-only, directories are writable\n" - "\n" - "general options:\n" - " -o opt,[opt...] mount options\n" - " -h --help print help\n" - "\n", progname); + "usage: %s basepath mountpoint [options]\n" + "\n" + " Makes basepath visible at mountpoint such that files are read-only, directories are writable\n" + "\n" + "general options:\n" + " -o opt,[opt...] mount options\n" + " -h --help print help\n" + "\n", progname); } static int rofs_parse_opt (void *data, const char *arg, int key, - struct fuse_args *outargs) + struct fuse_args *outargs) { (void) data; @@ -533,19 +532,19 @@ rofs_parse_opt (void *data, const char *arg, int key, { case FUSE_OPT_KEY_NONOPT: if (basefd == -1) - { - basefd = openat (AT_FDCWD, arg, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); - if (basefd == -1) - { - perror ("openat"); - exit (EXIT_FAILURE); - } - return 0; - } + { + basefd = openat (AT_FDCWD, arg, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); + if (basefd == -1) + { + perror ("openat"); + exit (EXIT_FAILURE); + } + return 0; + } else - { - return 1; - } + { + return 1; + } case FUSE_OPT_KEY_OPT: return 1; case KEY_HELP: diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index ce48a91e..1dac984d 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -1,13 +1,13 @@ /* -*- c-file-style: "gnu" -*- * Switch to new root directory and start init. - * + * * Copyright 2011,2012,2013 Colin Walters * - * Based on code from util-linux/sys-utils/switch_root.c, + * Based on code from util-linux/sys-utils/switch_root.c, * Copyright 2002-2009 Red Hat, Inc. All rights reserved. * Authors: - * Peter Jones - * Jeremy Katz + * Peter Jones + * Jeremy Katz * * Relicensed with permission to LGPLv2+. * @@ -91,15 +91,15 @@ parse_ostree_cmdline (void) const char *next = strchr (iter, ' '); const char *next_nonspc = next; while (next_nonspc && *next_nonspc == ' ') - next_nonspc += 1; + next_nonspc += 1; if (strncmp (iter, "ostree=", strlen ("ostree=")) == 0) { - const char *start = iter + strlen ("ostree="); - if (next) - ret = strndup (start, next - start); - else - ret = strdup (start); - break; + const char *start = iter + strlen ("ostree="); + if (next) + ret = strndup (start, next - start); + else + ret = strdup (start); + break; } iter = next_nonspc; } @@ -115,7 +115,7 @@ static void touch_run_ostree (void) { int fd; - + fd = open ("/run/ostree-booted", O_CREAT | O_WRONLY | O_NOCTTY, 0640); /* We ignore failures here in case /run isn't mounted...not much we * can do about that, but we don't want to fail. @@ -239,11 +239,11 @@ main(int argc, char *argv[]) * later boot and `systemd-remount-fs.service`. */ if (path_is_on_readonly_fs (".")) - { - if (mount (".", ".", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) - err (EXIT_FAILURE, "failed to remount rootfs writable (for overlayfs)"); - } - + { + if (mount (".", ".", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) + err (EXIT_FAILURE, "failed to remount rootfs writable (for overlayfs)"); + } + if (mount ("overlay", "usr", "overlay", 0, usr_ovl_options) < 0) err (EXIT_FAILURE, "failed to mount /usr overlayfs"); } diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 589a3867..1b5bd05a 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -85,13 +85,13 @@ main(int argc, char *argv[]) if (S_ISLNK (stbuf.st_mode)) continue; if (mount (target, target, NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) - { + { /* Also ignore ENINVAL - if the target isn't a mountpoint * already, then assume things are OK. */ if (errno != EINVAL) err (EXIT_FAILURE, "failed to remount %s", target); - } + } } maybe_mount_tmpfs_on_var (); diff --git a/tests/readdir-rand.c b/tests/readdir-rand.c index afef387c..10bf8677 100644 --- a/tests/readdir-rand.c +++ b/tests/readdir-rand.c @@ -83,7 +83,7 @@ readdir (DIR *dirp) struct dirent *(*real_readdir)(DIR *dirp) = dlsym (RTLD_NEXT, READDIR); struct dirent *ret; gboolean cache_another = TRUE; - + ensure_initialized (); /* The core idea here is that each time through the loop, we read a @@ -101,40 +101,40 @@ readdir (DIR *dirp) errno = 0; ret = real_readdir (dirp); if (ret == NULL && errno != 0) - goto out; + goto out; g_mutex_lock (&direntcache_lock); de = g_hash_table_lookup (direntcache, dirp); if (ret) - { - if (g_random_boolean ()) - { - struct dirent *copy; - if (!de) - { - de = dir_entries_new (); - g_hash_table_insert (direntcache, dirp, de); - } - copy = g_memdup (ret, sizeof (struct dirent)); - g_ptr_array_add (de->entries, copy); - } - else - { - cache_another = FALSE; - } - } + { + if (g_random_boolean ()) + { + struct dirent *copy; + if (!de) + { + de = dir_entries_new (); + g_hash_table_insert (direntcache, dirp, de); + } + copy = g_memdup (ret, sizeof (struct dirent)); + g_ptr_array_add (de->entries, copy); + } + else + { + cache_another = FALSE; + } + } else - { - if (de && de->offset < de->entries->len) - { - ret = de->entries->pdata[de->offset]; - de->offset++; - } - cache_another = FALSE; - } + { + if (de && de->offset < de->entries->len) + { + ret = de->entries->pdata[de->offset]; + de->offset++; + } + cache_another = FALSE; + } g_mutex_unlock (&direntcache_lock); } - + out: return ret; } @@ -145,7 +145,7 @@ closedir (DIR *dirp) int (*real_closedir)(DIR *dirp) = dlsym (RTLD_NEXT, "closedir"); ensure_initialized (); - + g_mutex_lock (&direntcache_lock); g_hash_table_remove (direntcache, dirp); g_mutex_unlock (&direntcache_lock); @@ -170,7 +170,7 @@ seekdir (DIR *dirp, long loc) ensure_initialized (); - /* For now, crash if seekdir is called when we have cached entries. + /* For now, crash if seekdir is called when we have cached entries. * If some app wants to use this and seekdir() we can implement it. */ assert_no_cached_entries (dirp); diff --git a/tests/test-rollsum.c b/tests/test-rollsum.c index 37c8bab8..a6c1bebd 100644 --- a/tests/test-rollsum.c +++ b/tests/test-rollsum.c @@ -152,7 +152,7 @@ test_bupsplit_sum(void) sum1a = bupsplit_sum(buf, 0, BUP_SELFTEST_SIZE); sum1b = bupsplit_sum(buf, 1, BUP_SELFTEST_SIZE); sum2a = bupsplit_sum(buf, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE*5/2, - BUP_SELFTEST_SIZE - BUP_WINDOWSIZE); + BUP_SELFTEST_SIZE - BUP_WINDOWSIZE); sum2b = bupsplit_sum(buf, 0, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE); sum3a = bupsplit_sum(buf, 0, BUP_WINDOWSIZE+3); sum3b = bupsplit_sum(buf, 3, BUP_WINDOWSIZE+3); From a195888b0f74ae3739a1a18fc46d39ae4686c5aa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 May 2017 20:29:21 -0400 Subject: [PATCH 79/94] lib/checkout: Fix regression in subpath for regular files This is what caused the merge of https://github.com/projectatomic/rpm-ostree/pull/652 to blow up, since https://github.com/ostreedev/ostree/pull/848 landed right before we tried to merge it. When I was writing that PR I remember having an uncertain feeling since we were doing a `mkdirat` above, but at the time I thought we'd have test suite coverage...turns out we didn't. For backwards compatibility, we need to continue to do a `mkdirat` here of the parent. However...I can't think of a reason anyone would *want* that behavior. Hence, let's add a special trick - if the destination name is `.`, we skip `mkdirat()`. That way rpm-ostree for example can open a dfd for `/etc` and avoid the `mkdir`. Fold the subpath tests into `test-basic.sh` since it's not worth a separate file. Add a test case for checking out a file. Closes: #854 Approved by: jlebon --- Makefile-tests.am | 1 - src/libostree/ostree-repo-checkout.c | 30 ++++++++++++++++++++---- tests/basic-test.sh | 12 ++++++++++ tests/test-repo-checkout-subpath.sh | 35 ---------------------------- 4 files changed, 37 insertions(+), 41 deletions(-) delete mode 100755 tests/test-repo-checkout-subpath.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 335ba9b7..89675288 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -96,7 +96,6 @@ _installed_or_uninstalled_test_scripts = \ tests/test-admin-pull-deploy-split.sh \ tests/test-admin-locking.sh \ tests/test-admin-deploy-clean.sh \ - tests/test-repo-checkout-subpath.sh \ tests/test-reset-nonlinear.sh \ tests/test-oldstyle-partial.sh \ tests/test-delta.sh \ diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index f5ef9517..236f514b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -812,11 +812,31 @@ checkout_tree_at (OstreeRepo *self, /* Special case handling for subpath of a non-directory */ if (g_file_info_get_file_type (source_info) != G_FILE_TYPE_DIRECTORY) - return checkout_one_file_at (self, options, &state, - ostree_repo_file_get_checksum (source), - destination_parent_fd, - g_file_info_get_name (source_info), - cancellable, error); + { + /* For backwards compat reasons, we do a mkdir() here. However, as a + * special case to allow callers to directly check out files without an + * intermediate root directory, we will skip mkdirat() if + * `destination_name` == `.`, since obviously the current directory + * exists. + */ + int destination_dfd = destination_parent_fd; + glnx_fd_close int destination_dfd_owned = -1; + if (strcmp (destination_name, ".") != 0) + { + if (mkdirat (destination_parent_fd, destination_name, 0700) < 0 + && errno != EEXIST) + return glnx_throw_errno_prefix (error, "mkdirat"); + if (!glnx_opendirat (destination_parent_fd, destination_name, TRUE, + &destination_dfd_owned, error)) + return FALSE; + destination_dfd = destination_dfd_owned; + } + return checkout_one_file_at (self, options, &state, + ostree_repo_file_get_checksum (source), + destination_dfd, + g_file_info_get_name (source_info), + cancellable, error); + } /* Cache any directory metadata we read during this operation; * see commit b7afe91e21143d7abb0adde440683a52712aa246 diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 6ddf7b2e..b209b839 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -341,6 +341,18 @@ cd ${test_tmpdir} $OSTREE checkout --subpath /yet/another test2 checkout-test2-subpath cd checkout-test2-subpath assert_file_has_content tree/green "leaf" +cd ${test_tmpdir} +rm checkout-test2-subpath -rf +$OSTREE ls -R test2 +# Test checking out a file +$OSTREE checkout --subpath /baz/saucer test2 checkout-test2-subpath +assert_file_has_content checkout-test2-subpath/saucer alien +# Test checking out a file without making a subdir +mkdir t +cd t +$OSTREE checkout --subpath /baz/saucer test2 . +assert_file_has_content saucer alien +rm t -rf echo "ok checkout subpath" cd ${test_tmpdir} diff --git a/tests/test-repo-checkout-subpath.sh b/tests/test-repo-checkout-subpath.sh deleted file mode 100755 index 2da5adc2..00000000 --- a/tests/test-repo-checkout-subpath.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2011,2013 Colin Walters -# Copyright (C) 2014 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -set -euo pipefail - -. $(dirname $0)/libtest.sh - -setup_test_repository "bare" - -echo '1..1' - -repopath=${test_tmpdir}/ostree-srv/gnomerepo - -${CMD_PREFIX} ostree --repo=repo checkout -U --subpath=/ test2 checkedout - -${CMD_PREFIX} ostree --repo=repo checkout -U --subpath=/firstfile test2 checkedout2 - -echo "ok" From 5811d4e8a3af4196f65f8dd5353c18227a79f992 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 12 May 2017 14:14:22 -0400 Subject: [PATCH 80/94] tests/ci-commitmessage-submodules.sh: fix for RHCI Special-case when this script is run under RHCI, which will try to fetch the merge commit if possible. Use RHCI_COMMIT instead to refer to the actual PR/branch HEAD being evaluated. Use realpath to workaround the developer's git dir being in a symbolic link. Closes: #857 Approved by: cgwalters --- tests/ci-commitmessage-submodules.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/ci-commitmessage-submodules.sh b/tests/ci-commitmessage-submodules.sh index 2403579d..8ce077c3 100755 --- a/tests/ci-commitmessage-submodules.sh +++ b/tests/ci-commitmessage-submodules.sh @@ -1,5 +1,6 @@ #!/bin/bash set -euo pipefail + # Copyright 2017 Colin Walters # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) @@ -12,6 +13,10 @@ set -euo pipefail # It's very common for people to accidentally change submodules, and having this # requirement is a small hurdle to pass. +# if running under RHCI, use the branch/PR HEAD actually +# being tested rather than the merge sha +HEAD=${RHCI_COMMIT:-HEAD} + tmpd=$(mktemp -d) touch ${tmpd}/.tmpdir cleanup_tmp() { @@ -22,13 +27,13 @@ cleanup_tmp() { } trap cleanup_tmp EXIT -gitdir=$(pwd) +gitdir=$(realpath $(pwd)) # Create a temporary copy of this (using cp not git clone) so git doesn't # try to read the submodules from the Internet again. If we wanted to # require a newer git, we could use `git worktree`. cp -a ${gitdir} ${tmpd}/workdir cd ${tmpd}/workdir -git log --pretty=oneline origin/master.. | while read logline; do +git log --pretty=oneline origin/master..$HEAD | while read logline; do commit=$(echo ${logline} | cut -f 1 -d ' ') git diff --name-only ${commit}^..${commit} > ${tmpd}/diff.txt git log -1 ${commit} > ${tmpd}/log.txt From 23c60cda22f27e76fe42b883786ed3955e5961f4 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 12 May 2017 14:17:21 -0400 Subject: [PATCH 81/94] libglnx: bump and use new helper methods Update submodule: libglnx Closes: #857 Approved by: cgwalters --- libglnx | 2 +- src/libostree/ostree-repo-checkout.c | 6 +++--- src/libostree/ostree-repo-commit.c | 4 ++-- src/libostree/ostree-repo.c | 4 ++-- src/libostree/ostree-sepolicy.c | 2 +- src/libostree/ostree-sysroot-deploy.c | 4 ++-- src/libostree/ostree-sysroot.c | 4 ++-- src/ostree/ot-admin-builtin-undeploy.c | 2 +- src/ostree/ot-main.c | 2 +- tests/libostreetest.c | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/libglnx b/libglnx index 3a4d0f46..32231fdb 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 3a4d0f4684f4653338c4756c8a1abfc6b5738467 +Subproject commit 32231fdb5273dd2a812c61549d6c0e681c0f5d59 diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 236f514b..f6e91498 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -516,7 +516,7 @@ checkout_one_file_at (OstreeRepo *repo, if (!checkout_object_for_uncompressed_cache (repo, loose_path_buf, source_info, input, cancellable, error)) - return g_prefix_error (error, "Unpacking loose object %s: ", checksum), FALSE; + return glnx_prefix_error (error, "Unpacking loose object %s", checksum); g_clear_object (&input); @@ -549,7 +549,7 @@ checkout_one_file_at (OstreeRepo *repo, destination_dfd, destination_name, FALSE, &hardlink_res, cancellable, error)) - return g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name), FALSE; + return glnx_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s", checksum, destination_name); need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } @@ -571,7 +571,7 @@ checkout_one_file_at (OstreeRepo *repo, if (!create_file_copy_from_input_at (repo, options, state, source_info, xattrs, input, destination_dfd, destination_name, cancellable, error)) - return g_prefix_error (error, "Copy checkout of %s to %s: ", checksum, destination_name), FALSE; + return glnx_prefix_error (error, "Copy checkout of %s to %s", checksum, destination_name); if (input) { diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 2e9f9243..ce45a911 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2009,13 +2009,13 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo *self, !ot_util_variant_map_at (self->commit_stagedir_fd, buf, G_VARIANT_TYPE ("a{sv}"), OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error)) - return g_prefix_error (error, "Unable to read existing detached metadata: "), FALSE; + return glnx_prefix_error (error, "Unable to read existing detached metadata"); if (ret_metadata == NULL && !ot_util_variant_map_at (self->objects_dir_fd, buf, G_VARIANT_TYPE ("a{sv}"), OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error)) - return g_prefix_error (error, "Unable to read existing detached metadata: "), FALSE; + return glnx_prefix_error (error, "Unable to read existing detached metadata"); if (ret_metadata == NULL && self->parent_repo) return ostree_repo_read_commit_detached_metadata (self->parent_repo, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 93e1888e..b7d1324c 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3917,7 +3917,7 @@ ostree_repo_sign_commit (OstreeRepo *self, g_autoptr(GVariant) commit_variant = NULL; if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit_checksum, &commit_variant, error)) - return g_prefix_error (error, "Failed to read commit: "), FALSE; + return glnx_prefix_error (error, "Failed to read commit"); g_autoptr(GVariant) old_metadata = NULL; if (!ostree_repo_read_commit_detached_metadata (self, @@ -3925,7 +3925,7 @@ ostree_repo_sign_commit (OstreeRepo *self, &old_metadata, cancellable, error)) - return g_prefix_error (error, "Failed to read detached metadata: "), FALSE; + return glnx_prefix_error (error, "Failed to read detached metadata"); commit_data = g_variant_get_data_as_bytes (commit_variant); diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index dd0567aa..7387ea0f 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -383,7 +383,7 @@ initable_init (GInitable *initable, freecon (con); if (!get_policy_checksum (&self->selinux_policy_csum, cancellable, error)) - return g_prefix_error (error, "While calculating SELinux checksum: "), FALSE; + return glnx_prefix_error (error, "While calculating SELinux checksum"); self->selinux_policy_name = g_steal_pointer (&policytype); self->selinux_policy_root = g_object_ref (etc_selinux_dir); diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 57020fa8..b4a8ec0c 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1306,7 +1306,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, contents = glnx_file_get_contents_utf8_at (deployment_dfd, "etc/os-release", NULL, cancellable, error); if (!contents) - return g_prefix_error (error, "Reading /etc/os-release: "), FALSE; + return glnx_prefix_error (error, "Reading /etc/os-release"); } } else @@ -1314,7 +1314,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, contents = glnx_file_get_contents_utf8_at (deployment_dfd, "usr/lib/os-release", NULL, cancellable, error); if (!contents) - return g_prefix_error (error, "Reading /usr/lib/os-release: "), FALSE; + return glnx_prefix_error (error, "Reading /usr/lib/os-release"); } g_autoptr(GHashTable) osrelease_values = parse_os_release (contents, "\n"); diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 67f940a8..510f2fb2 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -459,7 +459,7 @@ _ostree_sysroot_read_boot_loader_configs (OstreeSysroot *self, glnx_unref_object OstreeBootconfigParser *config = ostree_bootconfig_parser_new (); if (!ostree_bootconfig_parser_parse_at (config, dfd_iter.fd, dent->d_name, cancellable, error)) - return g_prefix_error (error, "Parsing %s: ", dent->d_name), FALSE; + return glnx_prefix_error (error, "Parsing %s", dent->d_name); g_ptr_array_add (ret_loader_configs, g_object_ref (config)); } @@ -533,7 +533,7 @@ parse_origin (OstreeSysroot *self, return FALSE; if (!g_key_file_load_from_data (ret_origin, origin_contents, -1, 0, error)) - return g_prefix_error (error, "Parsing %s: ", origin_path), FALSE; + return glnx_prefix_error (error, "Parsing %s", origin_path); } ot_transfer_out_value(out_origin, &ret_origin); diff --git a/src/ostree/ot-admin-builtin-undeploy.c b/src/ostree/ot-admin-builtin-undeploy.c index 4bf41d0c..00252c80 100644 --- a/src/ostree/ot-admin-builtin-undeploy.c +++ b/src/ostree/ot-admin-builtin-undeploy.c @@ -83,7 +83,7 @@ ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GEr ostree_deployment_get_deployserial (target_deployment)); if (!ostree_sysroot_cleanup (sysroot, cancellable, error)) - return g_prefix_error (error, "Performing final cleanup: "), FALSE; + return glnx_prefix_error (error, "Performing final cleanup"); return TRUE; } diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index a5b630a0..9aca8287 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -377,7 +377,7 @@ ostree_ensure_repo_writable (OstreeRepo *repo, GError **error) { if (!ostree_repo_is_writable (repo, error)) - return g_prefix_error (error, "Cannot write to repository: "), FALSE; + return glnx_prefix_error (error, "Cannot write to repository"); return TRUE; } diff --git a/tests/libostreetest.c b/tests/libostreetest.c index cda1649d..2fb83f51 100644 --- a/tests/libostreetest.c +++ b/tests/libostreetest.c @@ -99,7 +99,7 @@ ot_test_setup_sysroot (GCancellable *cancellable, { g_autoptr(GString) buf = g_string_new ("mutable-deployments"); if (statfs ("/", &stbuf) < 0) - return glnx_throw_errno (error), NULL; + return glnx_null_throw_errno (error); /* Keep this in sync with the overlayfs bits in libtest.sh */ #ifndef OVERLAYFS_SUPER_MAGIC #define OVERLAYFS_SUPER_MAGIC 0x794c7630 From 05d0ee5cbecd1287b87d38e969862a5d8b1f2e58 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 12 May 2017 15:34:52 -0400 Subject: [PATCH 82/94] remount: Drop support for auto-tmpfs-on-var; use systemd.volatile=state In current systemd, there is: [systemd-volatile-root](https://www.freedesktop.org/software/systemd/man/systemd-volatile-root.service.html) which was introduced by [this commit](https://github.com/systemd/systemd/commit/91214a37ef4eb8042d2598aa89bae52b410d11a7). I'd like to make further changes to how we handle `/var`, and I don't want to reason about the interaction of our "tmpfs var" with too many other things. The comment about having "all /var handling in one place" was always inaccurate given that we rely on systemd for mounting. And in general, I don't want to duplicate too many things systemd does - it does them well, documents them, etc. As far as I know, it was basically just Owen who was using this for the GNOME hardware testing effort, and I'm sure he could easily switch over to `systemd.volatile=state`. Closes: #856 Approved by: owtaylor --- src/switchroot/ostree-remount.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 1b5bd05a..bd4d1253 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -38,24 +38,6 @@ #include "ostree-mount-util.h" -/* Having a writeable /var is necessary for full system functioning. - * If /var isn't writeable, we mount tmpfs over it. While this is - * somewhat outside of ostree's scope, having all /var twiddling - * in one place will make future maintenance easier. - */ -static void -maybe_mount_tmpfs_on_var (void) -{ - if (!path_is_on_readonly_fs ("/var")) - return; - - if (umount ("/var") < 0 && errno != EINVAL) - warn ("failed to unmount /var prior to mounting tmpfs, mounting over"); - - if (mount ("tmpfs", "/var", "tmpfs", 0, NULL) < 0) - err (EXIT_FAILURE, "failed to mount tmpfs on /var"); -} - int main(int argc, char *argv[]) { @@ -69,8 +51,6 @@ main(int argc, char *argv[]) * to clear the readonly flag in that case. */ - maybe_mount_tmpfs_on_var (); - exit (EXIT_SUCCESS); } @@ -94,7 +74,5 @@ main(int argc, char *argv[]) } } - maybe_mount_tmpfs_on_var (); - exit (EXIT_SUCCESS); } From a5eef45debcf8d2550deef8c5b9f33956d786fe9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 15 May 2017 09:10:54 -0400 Subject: [PATCH 83/94] lib/remote: Box OstreeRemote if experimental-api To avoid an introspection warning. Otherwise, don't box it. Closes: #858 Approved by: pwithnall --- src/libostree/ostree-remote.c | 6 ++++++ src/libostree/ostree-remote.h | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c index d6298fba..86fae8d8 100644 --- a/src/libostree/ostree-remote.c +++ b/src/libostree/ostree-remote.c @@ -142,3 +142,9 @@ ostree_remote_unref (OstreeRemote *remote) g_slice_free (OstreeRemote, remote); } } + +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +G_DEFINE_BOXED_TYPE(OstreeRemote, ostree_remote, + ostree_remote_ref, + ostree_remote_unref); +#endif diff --git a/src/libostree/ostree-remote.h b/src/libostree/ostree-remote.h index bf62fd87..8e96213c 100644 --- a/src/libostree/ostree-remote.h +++ b/src/libostree/ostree-remote.h @@ -48,9 +48,16 @@ G_BEGIN_DECLS typedef struct OstreeRemote OstreeRemote; #endif +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +_OSTREE_PUBLIC +GType ostree_remote_get_type (void) G_GNUC_CONST; +#else +#ifndef __GI_SCANNER__ _OSTREE_PUBLIC OstreeRemote *ostree_remote_ref (OstreeRemote *remote); _OSTREE_PUBLIC void ostree_remote_unref (OstreeRemote *remote); +#endif /* GI_SCANNER */ +#endif G_END_DECLS From 19827a996597a8df2ddbb344e4da1661488df99c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 09:57:31 -0400 Subject: [PATCH 84/94] lib/repo: Fix double close() Should probably change `_take_fd()` to take a pointer and set to `-1` at some point. Regression from 8d58ab1002cbc4a1ecafe3d1a80984f8a60f41e9 Closes: #862 Approved by: jlebon --- src/libostree/ostree-repo.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b7d1324c..2b164ffb 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2283,6 +2283,7 @@ list_loose_objects_at (OstreeRepo *self, g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_take_fd (target_dfd, &dfd_iter, error)) return FALSE; + target_dfd = -1; /* Transferred */ while (TRUE) { From f21f500e40ba0d2c0881ecdaf876f8510e8c64ba Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 12 May 2017 15:55:12 -0400 Subject: [PATCH 85/94] switchroot/remount: Trim set of remounted filesystems I really have no idea what I was thinking with that list of mount points. It seems arbitrary. Sadly `git log` doesn't help, and there's no comments. Basically, the only mounts we should care about are those that libostree creates. Which are just `/sysroot` and `/var`. Systemd will handle the other things like `/tmp`, it's not our job, and we shouldn't touch them. Closes: #859 Approved by: jlebon --- src/switchroot/ostree-remount.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index bd4d1253..e19de183 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) { - const char *remounts[] = { "/sysroot", "/etc", "/home", "/root", "/tmp", "/var", NULL }; + const char *remounts[] = { "/sysroot", "/var", NULL }; struct stat stbuf; int i; From d815ba2a81ad14d9d4edc31dcd282dcc2a3a8fb9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 15 May 2017 13:59:57 -0400 Subject: [PATCH 86/94] switchroot/remount: Check mount status before remounting, be verbose By checking the mount status, we avoid remounting things if we don't need to. And printing a single line per mount helps debugging when things go wrong. Closes: #859 Approved by: jlebon --- src/switchroot/ostree-remount.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index e19de183..a6d14d08 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -64,6 +64,14 @@ main(int argc, char *argv[]) */ if (S_ISLNK (stbuf.st_mode)) continue; + /* If not a mountpoint, skip it */ + struct statvfs stvfsbuf; + if (statvfs (target, &stvfsbuf) == -1) + continue; + /* If no read-only flag, skip it */ + if ((stvfsbuf.f_flag & ST_RDONLY) == 0) + continue; + /* It's a mounted, read-only fs; remount it */ if (mount (target, target, NULL, MS_REMOUNT | MS_SILENT, NULL) < 0) { /* Also ignore ENINVAL - if the target isn't a mountpoint @@ -72,6 +80,8 @@ main(int argc, char *argv[]) if (errno != EINVAL) err (EXIT_FAILURE, "failed to remount %s", target); } + else + printf ("Remounted: %s\n", target); } exit (EXIT_SUCCESS); From 30705889cb867c18cdb7fed8e55dc46477c069fa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 May 2017 14:54:12 -0400 Subject: [PATCH 87/94] Switch to using a systemd generator for /var If one wants to set up a mount for `/var` in `/etc/fstab`, it won't be mounted since `ostree-prepare-root` set up a bind mount for `/var` to `/sysroot/ostree/$stateroot/var`, and systemd will take the already extant mount over what's in `/etc/fstab`. There are a few options to fix this, but what I settled on is parsing `/etc/fstab` in a generator (exactly like `systemd-fstab-generator` does), except here we look for an explicit mount for `/var`, and if one *isn't* found, synthesize the default ostree mount to the stateroot. Another nice property is that if an admin creates a `var.mount` unit in `/etc` for example, that will also override our mount. Note that today ostree doesn't hard depend on systemd, so this behavior only kicks in if we're built with systemd *and* libmount support (for parsing `/etc/fstab`). I didn't really test that case though. Initially I started writing this as a "pure libc" program, but at one point decided to use `libostree.so` to find the booted deployment. That didn't work out because `/boot` wasn't necessarily mounted and hence we couldn't find the bootloader config. A leftover artifact from this is that the generator code calls into libostree via the "cmd private" infrastructure. But it's an easy way to share code, and doesn't hurt. Closes: #859 Approved by: jlebon --- Makefile-libostree.am | 1 + Makefile-switchroot.am | 19 +- configure.ac | 11 + src/libostree/ostree-cmdprivate.c | 1 + src/libostree/ostree-cmdprivate.h | 3 + src/libostree/ostree-impl-system-generator.c | 219 +++++++++++++++++++ src/libostree/ostree-sysroot-deploy.c | 1 + src/switchroot/ostree-mount-util.h | 66 ++++++ src/switchroot/ostree-prepare-root.c | 67 +----- src/switchroot/ostree-system-generator.c | 67 ++++++ 10 files changed, 390 insertions(+), 65 deletions(-) create mode 100644 src/libostree/ostree-impl-system-generator.c create mode 100644 src/switchroot/ostree-system-generator.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 86ba0414..7f2e2a4a 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -115,6 +115,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-sysroot-cleanup.c \ src/libostree/ostree-sysroot-deploy.c \ src/libostree/ostree-sysroot-upgrader.c \ + src/libostree/ostree-impl-system-generator.c \ src/libostree/ostree-bootconfig-parser.c \ src/libostree/ostree-deployment.c \ src/libostree/ostree-bootloader.h \ diff --git a/Makefile-switchroot.am b/Makefile-switchroot.am index 6fd2f820..dd24010e 100644 --- a/Makefile-switchroot.am +++ b/Makefile-switchroot.am @@ -27,6 +27,7 @@ ostree_prepare_root_SOURCES = \ src/switchroot/ostree-mount-util.h \ src/switchroot/ostree-prepare-root.c \ $(NULL) +ostree_prepare_root_CPPFLAGS = $(AM_CPPFLAGS) if BUILDOPT_USE_STATIC_COMPILER # ostree-prepare-root can be used as init in a system without a populated /lib. @@ -45,7 +46,6 @@ ostree-prepare-root : $(ostree_prepare_root_SOURCES) $(STATIC_COMPILER) -o $@ -static $(ostree_prepare_root_SOURCES) $(AM_CPPFLAGS) $(AM_CFLAGS) $(DEFAULT_INCLUDES) else ostree_boot_PROGRAMS += ostree-prepare-root - ostree_prepare_root_CFLAGS = $(AM_CFLAGS) -Isrc/switchroot endif @@ -53,4 +53,19 @@ ostree_remount_SOURCES = \ src/switchroot/ostree-mount-util.h \ src/switchroot/ostree-remount.c \ $(NULL) -ostree_remount_CFLAGS = $(AM_CFLAGS) -Isrc/switchroot +ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) -Isrc/switchroot + +# This is the "new mode" of using a generator for /var; see +# https://github.com/ostreedev/ostree/issues/855 +if BUILDOPT_SYSTEMD_AND_LIBMOUNT +ostree_prepare_root_CPPFLAGS += -DHAVE_SYSTEMD_AND_LIBMOUNT=1 +ostree_remount_CPPFLAGS += -DHAVE_SYSTEMD_AND_LIBMOUNT=1 + +systemdsystemgenerator_PROGRAMS = ostree-system-generator +GITIGNOREFILES += $(systemdsystemgenerator_PROGRAMS) +ostree_system_generator_SOURCES = src/switchroot/ostree-mount-util.h \ + src/switchroot/ostree-system-generator.c +ostree_system_generator_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/libglnx -I$(srcdir)/src/libostree +ostree_system_generator_CFLAGS = $(AM_CFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) +ostree_system_generator_LDADD = $(AM_LDFLAGS) libglnx.la libostree-1.la $(OT_INTERNAL_GIO_UNIX_LIBS) +endif diff --git a/configure.ac b/configure.ac index 6acd0b3c..06d39126 100644 --- a/configure.ac +++ b/configure.ac @@ -389,8 +389,19 @@ AS_IF([test "x$have_libsystemd" = "xyes"], [ AS_IF([test "x$with_systemdsystemunitdir" != "xno"], [ AC_SUBST([systemdsystemunitdir], [$with_systemdsystemunitdir]) ]) + AC_ARG_WITH([systemdsystemgeneratordir], + AS_HELP_STRING([--with-systemdsystemgeneratordir=DIR], [Directory for systemd generators]), + [], + [with_systemdsystemgeneratordir=$($PKG_CONFIG --variable=systemdsystemgeneratordir systemd)]) + AS_IF([test "x$with_systemdsystemgeneratordir" != "xno"], [ + AC_SUBST([systemdsystemgeneratordir], [$with_systemdsystemgeneratordir]) + ]) ]) AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_systemd = xyes) +dnl If we have both, we use the "new /var" model with ostree-system-generator +AM_CONDITIONAL(BUILDOPT_SYSTEMD_AND_LIBMOUNT,[test x$with_systemd = xyes && test x$with_libmount = xyes]) +AM_COND_IF(BUILDOPT_SYSTEMD_AND_LIBMOUNT, + AC_DEFINE([BUILDOPT_LIBSYSTEMD_AND_LIBMOUNT], 1, [Define if systemd and libmount])) AC_ARG_WITH(builtin-grub2-mkconfig, AS_HELP_STRING([--with-builtin-grub2-mkconfig], diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index 4367b497..bade7431 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -45,6 +45,7 @@ const OstreeCmdPrivateVTable * ostree_cmd__private__ (void) { static OstreeCmdPrivateVTable table = { + _ostree_impl_system_generator, impl_ostree_generate_grub2_config, _ostree_repo_static_delta_dump, _ostree_repo_static_delta_query_exists, diff --git a/src/libostree/ostree-cmdprivate.h b/src/libostree/ostree-cmdprivate.h index 8d1c653e..63c427cd 100644 --- a/src/libostree/ostree-cmdprivate.h +++ b/src/libostree/ostree-cmdprivate.h @@ -24,7 +24,10 @@ G_BEGIN_DECLS +gboolean _ostree_impl_system_generator (const char *ostree_cmdline, const char *normal_dir, const char *early_dir, const char *late_dir, GError **error); + typedef struct { + gboolean (* ostree_system_generator) (const char *ostree_cmdline, const char *normal_dir, const char *early_dir, const char *late_dir, GError **error); gboolean (* ostree_generate_grub2_config) (OstreeSysroot *sysroot, int bootversion, int target_fd, GCancellable *cancellable, GError **error); gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); gboolean (* ostree_static_delta_query_exists) (OstreeRepo *repo, const char *delta_id, gboolean *out_exists, GCancellable *cancellable, GError **error); diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c new file mode 100644 index 00000000..7c4d49df --- /dev/null +++ b/src/libostree/ostree-impl-system-generator.c @@ -0,0 +1,219 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include +#include +#include +#include +#ifdef HAVE_LIBMOUNT +#include +#endif +#include +#include "otutil.h" + +#include "ostree.h" +#include "ostree-core-private.h" +#include "ostree-cmdprivate.h" + +#ifdef HAVE_LIBMOUNT +typedef FILE OtLibMountFile; +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtLibMountFile, endmntent); + +/* Taken from systemd path-util.c */ +static bool +is_path (const char *p) +{ + return !!strchr (p, '/'); +} + +/* Taken from systemd path-util.c */ +static char* +path_kill_slashes (char *path) +{ + char *f, *t; + bool slash = false; + + /* Removes redundant inner and trailing slashes. Modifies the + * passed string in-place. + * + * For example: ///foo///bar/ becomes /foo/bar + */ + + for (f = path, t = path; *f; f++) + { + if (*f == '/') + { + slash = true; + continue; + } + + if (slash) + { + slash = false; + *(t++) = '/'; + } + + *(t++) = *f; + } + + /* Special rule, if we are talking of the root directory, a + trailing slash is good */ + + if (t == path && slash) + *(t++) = '/'; + + *t = 0; + return path; +} + +/* Written by ostree-sysroot-deploy.c. We parse out the stateroot here since we + * need to know it to mount /var. Unfortunately we can't easily use the + * libostree API to find the booted deployment since /boot might not have been + * mounted yet. + */ +static char * +stateroot_from_ostree_cmdline (const char *ostree_cmdline, + GError **error) +{ + static GRegex *regex; + static gsize regex_initialized; + if (g_once_init_enter (®ex_initialized)) + { + regex = g_regex_new ("^/ostree/boot.[01]/([^/]+)/", 0, 0, NULL); + g_assert (regex); + g_once_init_leave (®ex_initialized, 1); + } + + g_autoptr(GMatchInfo) match = NULL; + if (!g_regex_match (regex, ostree_cmdline, 0, &match)) + return glnx_null_throw (error, "Failed to parse %s", ostree_cmdline); + + return g_match_info_fetch (match, 1); +} +#endif + +/* Implementation of ostree-system-generator */ +gboolean +_ostree_impl_system_generator (const char *ostree_cmdline, + const char *normal_dir, + const char *early_dir, + const char *late_dir, + GError **error) +{ +#ifdef HAVE_LIBMOUNT + /* Not currently cancellable, but define a var in case we care later */ + GCancellable *cancellable = NULL; + /* Some path constants to avoid typos */ + static const char fstab_path[] = "/etc/fstab"; + static const char var_path[] = "/var"; + + /* ostree-prepare-root was patched to write the stateroot to this file */ + g_autofree char *stateroot = stateroot_from_ostree_cmdline (ostree_cmdline, error); + if (!stateroot) + return FALSE; + + /* Load /etc/fstab if it exists, and look for a /var mount */ + g_autoptr(OtLibMountFile) fstab = setmntent (fstab_path, "re"); + gboolean found_var_mnt = FALSE; + if (!fstab) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "Reading %s", fstab_path); + } + else + { + /* Parse it */ + struct mntent *me; + while ((me = getmntent (fstab))) + { + g_autofree char *where = g_strdup (me->mnt_dir); + if (is_path (where)) + path_kill_slashes (where); + + /* We're only looking for /var here */ + if (strcmp (where, var_path) != 0) + continue; + + found_var_mnt = TRUE; + break; + } + } + + /* If we found /var, we're done */ + if (found_var_mnt) + return TRUE; + + /* Prepare to write to the output unit dir; we use the "normal" dir + * that overrides /usr, but not /etc. + */ + glnx_fd_close int normal_dir_dfd = -1; + if (!glnx_opendirat (AT_FDCWD, normal_dir, TRUE, &normal_dir_dfd, error)) + return FALSE; + + /* Generate our bind mount unit */ + const char *stateroot_var_path = glnx_strjoina ("/sysroot/ostree/deploy/", stateroot, "/var"); + + glnx_fd_close int tmpfd = -1; + g_autofree char *tmppath = NULL; + if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY, + &tmpfd, &tmppath, error)) + return FALSE; + g_autoptr(GOutputStream) outstream = g_unix_output_stream_new (tmpfd, FALSE); + gsize bytes_written; + /* This code is inspired by systemd's fstab-generator.c. + * + * Note that our unit doesn't run if systemd.volatile is enabled; + * see https://github.com/ostreedev/ostree/pull/856 + */ + if (!g_output_stream_printf (outstream, &bytes_written, cancellable, error, + "##\n# Automatically generated by ostree-system-generator\n##\n\n" + "[Unit]\n" + "Documentation=man:ostree(1)\n" + "ConditionKernelCommandLine=!systemd.volatile\n" + /* We need /sysroot mounted writable first */ + "After=ostree-remount.service\n" + "Before=local-fs.target\n" + "\n" + "[Mount]\n" + "Where=%s\n" + "What=%s\n" + "Options=bind\n", + var_path, + stateroot_var_path)) + return FALSE; + if (!g_output_stream_flush (outstream, cancellable, error)) + return FALSE; + g_clear_object (&outstream); + /* It should be readable */ + if (fchmod (tmpfd, 0644) < 0) + return glnx_throw_errno_prefix (error, "fchmod"); + /* Error out if somehow it already exists, that'll help us debug conflicts */ + if (!glnx_link_tmpfile_at (normal_dir_dfd, GLNX_LINK_TMPFILE_NOREPLACE, + tmpfd, tmppath, normal_dir_dfd, + "var.mount", error)) + return FALSE; + + return TRUE; +#else + return glnx_throw (error, "Not implemented"); +#endif +} diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index b4a8ec0c..257a058b 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -1379,6 +1379,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, val = ostree_bootconfig_parser_get (bootconfig, "options"); + /* Note this is parsed in ostree-impl-system-generator.c */ g_autofree char *ostree_kernel_arg = g_strdup_printf ("ostree=/ostree/boot.%d/%s/%s/%d", new_bootversion, osname, bootcsum, ostree_deployment_get_bootserial (deployment)); diff --git a/src/switchroot/ostree-mount-util.h b/src/switchroot/ostree-mount-util.h index b24aa44d..1e7253d2 100644 --- a/src/switchroot/ostree-mount-util.h +++ b/src/switchroot/ostree-mount-util.h @@ -25,6 +25,10 @@ #include #include #include +#include +#include +#include +#include static inline int path_is_on_readonly_fs (char *path) @@ -37,4 +41,66 @@ path_is_on_readonly_fs (char *path) return (stvfsbuf.f_flag & ST_RDONLY) != 0; } +static inline char * +read_proc_cmdline (void) +{ + FILE *f = fopen("/proc/cmdline", "r"); + char *cmdline = NULL; + size_t len; + + if (!f) + goto out; + + /* Note that /proc/cmdline will not end in a newline, so getline + * will fail unelss we provide a length. + */ + if (getline (&cmdline, &len, f) < 0) + goto out; + /* ... but the length will be the size of the malloc buffer, not + * strlen(). Fix that. + */ + len = strlen (cmdline); + + if (cmdline[len-1] == '\n') + cmdline[len-1] = '\0'; +out: + if (f) + fclose (f); + return cmdline; +} + +static inline char * +read_proc_cmdline_ostree (void) +{ + char *cmdline = NULL; + const char *iter; + char *ret = NULL; + + cmdline = read_proc_cmdline (); + if (!cmdline) + err (EXIT_FAILURE, "failed to read /proc/cmdline"); + + iter = cmdline; + while (iter != NULL) + { + const char *next = strchr (iter, ' '); + const char *next_nonspc = next; + while (next_nonspc && *next_nonspc == ' ') + next_nonspc += 1; + if (strncmp (iter, "ostree=", strlen ("ostree=")) == 0) + { + const char *start = iter + strlen ("ostree="); + if (next) + ret = strndup (start, next - start); + else + ret = strdup (start); + break; + } + iter = next_nonspc; + } + + free (cmdline); + return ret; +} + #endif /* __OSTREE_MOUNT_UTIL_H_ */ diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 1dac984d..15dbafdf 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -46,68 +46,6 @@ #include "ostree-mount-util.h" -static char * -read_proc_cmdline (void) -{ - FILE *f = fopen("/proc/cmdline", "r"); - char *cmdline = NULL; - size_t len; - - if (!f) - goto out; - - /* Note that /proc/cmdline will not end in a newline, so getline - * will fail unelss we provide a length. - */ - if (getline (&cmdline, &len, f) < 0) - goto out; - /* ... but the length will be the size of the malloc buffer, not - * strlen(). Fix that. - */ - len = strlen (cmdline); - - if (cmdline[len-1] == '\n') - cmdline[len-1] = '\0'; -out: - if (f) - fclose (f); - return cmdline; -} - -static char * -parse_ostree_cmdline (void) -{ - char *cmdline = NULL; - const char *iter; - char *ret = NULL; - - cmdline = read_proc_cmdline (); - if (!cmdline) - err (EXIT_FAILURE, "failed to read /proc/cmdline"); - - iter = cmdline; - while (iter != NULL) - { - const char *next = strchr (iter, ' '); - const char *next_nonspc = next; - while (next_nonspc && *next_nonspc == ' ') - next_nonspc += 1; - if (strncmp (iter, "ostree=", strlen ("ostree=")) == 0) - { - const char *start = iter + strlen ("ostree="); - if (next) - ret = strndup (start, next - start); - else - ret = strdup (start); - break; - } - iter = next_nonspc; - } - - free (cmdline); - return ret; -} - /* This is an API for other projects to determine whether or not the * currently running system is ostree-controlled. */ @@ -132,7 +70,7 @@ resolve_deploy_path (const char * root_mountpoint) struct stat stbuf; char *ostree_target, *deploy_path; - ostree_target = parse_ostree_cmdline (); + ostree_target = read_proc_cmdline_ostree (); if (!ostree_target) errx (EXIT_FAILURE, "No OSTree target; expected ostree=/ostree/boot.N/..."); @@ -211,9 +149,12 @@ main(int argc, char *argv[]) if (chdir (deploy_path) < 0) err (EXIT_FAILURE, "failed to chdir to deploy_path"); + /* In the systemd case, this is handled by ostree-system-generator */ +#ifndef HAVE_SYSTEMD_AND_LIBMOUNT /* Link to the deployment's /var */ if (mount ("../../var", "var", NULL, MS_MGC_VAL|MS_BIND, NULL) < 0) err (EXIT_FAILURE, "failed to bind mount ../../var to var"); +#endif /* If /boot is on the same partition, use a bind mount to make it visible * at /boot inside the deployment. */ diff --git a/src/switchroot/ostree-system-generator.c b/src/switchroot/ostree-system-generator.c new file mode 100644 index 00000000..e7205b5a --- /dev/null +++ b/src/switchroot/ostree-system-generator.c @@ -0,0 +1,67 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-cmdprivate.h" +#include "ostree-mount-util.h" + +static const char *arg_dest = "/tmp"; +static const char *arg_dest_late = "/tmp"; + +/* This program is a simple stub that calls the implementation that + * lives inside libostree. + */ +int +main(int argc, char *argv[]) +{ + /* Important: if this isn't an ostree-booted system, do nothing; people could + * have the package installed as a dependency for flatpak or whatever. + */ + { struct stat stbuf; + if (fstatat (AT_FDCWD, "/run/ostree-booted", &stbuf, 0) < 0) + exit (EXIT_SUCCESS); + } + + if (argc > 1 && argc != 4) + errx (EXIT_FAILURE, "This program takes three or no arguments"); + + if (argc > 1) + arg_dest = argv[1]; + if (argc > 3) + arg_dest_late = argv[3]; + + char *ostree_cmdline = read_proc_cmdline_ostree (); + if (!ostree_cmdline) + errx (EXIT_FAILURE, "Failed to find ostree= kernel argument"); + + { g_autoptr(GError) local_error = NULL; + if (!ostree_cmd__private__()->ostree_system_generator (ostree_cmdline, arg_dest, NULL, arg_dest_late, &local_error)) + errx (EXIT_FAILURE, "%s", local_error->message); + } + + exit (EXIT_SUCCESS); +} From 90cd7f72344fc84da9d1f73189bdeebdcc4596aa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 12:42:45 -0400 Subject: [PATCH 88/94] tree-wide: Add a few missing O_CLOEXEC I noticed an instance of this while working on https://github.com/ostreedev/ostree/pull/861 Which apparently I cargo-culted into the new system generator bits. Let's break this out as a small concise change. Closes: #866 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 2 +- src/libostree/ostree-impl-system-generator.c | 2 +- src/switchroot/ostree-prepare-root.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 829f4447..f6893fd0 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -272,7 +272,7 @@ ensure_tmpfile (FetcherRequest *req, GError **error) if (req->out_tmpfile_fd == -1) { if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", - O_WRONLY, &req->out_tmpfile_fd, + O_WRONLY | O_CLOEXEC, &req->out_tmpfile_fd, &req->out_tmpfile, error)) return FALSE; diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c index 7c4d49df..60df145a 100644 --- a/src/libostree/ostree-impl-system-generator.c +++ b/src/libostree/ostree-impl-system-generator.c @@ -174,7 +174,7 @@ _ostree_impl_system_generator (const char *ostree_cmdline, glnx_fd_close int tmpfd = -1; g_autofree char *tmppath = NULL; - if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY, + if (!glnx_open_tmpfile_linkable_at (normal_dir_dfd, ".", O_WRONLY | O_CLOEXEC, &tmpfd, &tmppath, error)) return FALSE; g_autoptr(GOutputStream) outstream = g_unix_output_stream_new (tmpfd, FALSE); diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 15dbafdf..9b8c3381 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -54,7 +54,7 @@ touch_run_ostree (void) { int fd; - fd = open ("/run/ostree-booted", O_CREAT | O_WRONLY | O_NOCTTY, 0640); + fd = open ("/run/ostree-booted", O_CREAT | O_WRONLY | O_NOCTTY | O_CLOEXEC, 0640); /* We ignore failures here in case /run isn't mounted...not much we * can do about that, but we don't want to fail. */ From 9380dbb14d29de77a9fdd0b7bd7ff63bb0e0d441 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 10:51:40 -0400 Subject: [PATCH 89/94] lib: Add "open dfd iter handling noent" helper, port tree-wide Follow up to a previous patch that addressed a double-close; I realized we already had a helper for doing "open dfd iter, do nothing if we get ENOENT". Raise it to libotuil, and port all consumers. Closes: #863 Approved by: jlebon --- src/libostree/ostree-repo-prune.c | 19 +-- src/libostree/ostree-repo-static-delta-core.c | 124 ++++++++---------- src/libostree/ostree-repo.c | 15 +-- src/libostree/ostree-sysroot-cleanup.c | 29 +--- src/libostree/ostree-sysroot.c | 24 ++-- src/libotutil/ot-fs-utils.c | 25 ++++ src/libotutil/ot-fs-utils.h | 7 + 7 files changed, 112 insertions(+), 131 deletions(-) diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 76d64794..bf0a2530 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -114,19 +114,14 @@ _ostree_repo_prune_tmp (OstreeRepo *self, if (self->cache_dir_fd == -1) return TRUE; - glnx_fd_close int fd = glnx_opendirat_with_errno (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, FALSE); - if (fd < 0) - { - /* Note early return */ - if (errno == ENOENT) - return TRUE; - else - return glnx_throw_errno_prefix (error, "opendirat(%s)", _OSTREE_SUMMARY_CACHE_DIR); - } - g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!glnx_dirfd_iterator_init_take_fd (dup (fd), &dfd_iter, error)) + gboolean exists; + if (!ot_dfd_iter_init_allow_noent (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, + &dfd_iter, &exists, error)) return FALSE; + /* Note early return */ + if (!exists) + return TRUE; while (TRUE) { @@ -152,7 +147,7 @@ _ostree_repo_prune_tmp (OstreeRepo *self, if (has_sig_suffix) dent->d_name[len - 4] = '.'; - if (unlinkat (fd, dent->d_name, 0) < 0) + if (unlinkat (dfd_iter.fd, dent->d_name, 0) < 0) return glnx_throw_errno_prefix (error, "unlinkat"); } } diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index 21ed081d..3a14f8bb 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -73,99 +73,89 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(GPtrArray) ret_deltas = NULL; - glnx_fd_close int dfd = -1; + g_autoptr(GPtrArray) ret_deltas = g_ptr_array_new_with_free_func (g_free); - ret_deltas = g_ptr_array_new_with_free_func (g_free); - - dfd = glnx_opendirat_with_errno (self->repo_dir_fd, "deltas", TRUE); - if (dfd < 0) + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + gboolean exists; + if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "deltas", &dfd_iter, + &exists, error)) + return FALSE; + if (!exists) { - if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - return FALSE; - } + /* Note early return */ + ot_transfer_out_value (out_deltas, &ret_deltas); + return TRUE; } - else - { - g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!glnx_dirfd_iterator_init_take_fd (dfd, &dfd_iter, error)) + while (TRUE) + { + g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, }; + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + if (dent->d_type != DT_DIR) + continue; + + if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, + &sub_dfd_iter, error)) return FALSE; - dfd = -1; while (TRUE) { - g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, }; - struct dirent *dent; + struct dirent *sub_dent; + const char *name1; + const char *name2; + g_autofree char *superblock_subpath = NULL; + struct stat stbuf; - if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent, + cancellable, error)) return FALSE; - if (dent == NULL) + if (sub_dent == NULL) break; if (dent->d_type != DT_DIR) continue; - if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, - &sub_dfd_iter, error)) - return FALSE; + name1 = dent->d_name; + name2 = sub_dent->d_name; - while (TRUE) + superblock_subpath = g_strconcat (name2, "/superblock", NULL); + if (fstatat (sub_dfd_iter.fd, superblock_subpath, &stbuf, 0) < 0) { - struct dirent *sub_dent; - const char *name1; - const char *name2; - g_autofree char *superblock_subpath = NULL; - struct stat stbuf; - - if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent, - cancellable, error)) - return FALSE; - if (sub_dent == NULL) - break; - if (dent->d_type != DT_DIR) - continue; - - name1 = dent->d_name; - name2 = sub_dent->d_name; - - superblock_subpath = g_strconcat (name2, "/superblock", NULL); - if (fstatat (sub_dfd_iter.fd, superblock_subpath, &stbuf, 0) < 0) + if (errno != ENOENT) { - if (errno != ENOENT) - { - glnx_set_error_from_errno (error); - return FALSE; - } + glnx_set_error_from_errno (error); + return FALSE; } - else - { - g_autofree char *buf = g_strconcat (name1, name2, NULL); - GString *out = g_string_new (""); - char checksum[OSTREE_SHA256_STRING_LEN+1]; - guchar csum[OSTREE_SHA256_DIGEST_LEN]; - const char *dash = strchr (buf, '-'); + } + else + { + g_autofree char *buf = g_strconcat (name1, name2, NULL); + GString *out = g_string_new (""); + char checksum[OSTREE_SHA256_STRING_LEN+1]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; + const char *dash = strchr (buf, '-'); - ostree_checksum_b64_inplace_to_bytes (buf, csum); + ostree_checksum_b64_inplace_to_bytes (buf, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + g_string_append (out, checksum); + if (dash) + { + g_string_append_c (out, '-'); + ostree_checksum_b64_inplace_to_bytes (dash+1, csum); ostree_checksum_inplace_from_bytes (csum, checksum); g_string_append (out, checksum); - if (dash) - { - g_string_append_c (out, '-'); - ostree_checksum_b64_inplace_to_bytes (dash+1, csum); - ostree_checksum_inplace_from_bytes (csum, checksum); - g_string_append (out, checksum); - } - - g_ptr_array_add (ret_deltas, g_string_free (out, FALSE)); } + + g_ptr_array_add (ret_deltas, g_string_free (out, FALSE)); } } } - if (out_deltas) - *out_deltas = g_steal_pointer (&ret_deltas); + ot_transfer_out_value (out_deltas, &ret_deltas); return TRUE; } diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2b164ffb..2ef64ec0 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2272,18 +2272,13 @@ list_loose_objects_at (OstreeRepo *self, { GVariant *key, *value; - glnx_fd_close int target_dfd = glnx_opendirat_with_errno (dfd, prefix, FALSE); - if (target_dfd < 0) - { - /* Nothing to do if this dir doesn't exist */ - if (errno == ENOENT) - return TRUE; - return glnx_throw_errno (error); - } g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!glnx_dirfd_iterator_init_take_fd (target_dfd, &dfd_iter, error)) + gboolean exists; + if (!ot_dfd_iter_init_allow_noent (dfd, prefix, &dfd_iter, &exists, error)) return FALSE; - target_dfd = -1; /* Transferred */ + /* Note early return */ + if (!exists) + return TRUE; while (TRUE) { diff --git a/src/libostree/ostree-sysroot-cleanup.c b/src/libostree/ostree-sysroot-cleanup.c index 2933d459..499f20eb 100644 --- a/src/libostree/ostree-sysroot-cleanup.c +++ b/src/libostree/ostree-sysroot-cleanup.c @@ -25,31 +25,6 @@ #include "ostree-sysroot-private.h" -/* Like glnx_dirfd_iterator_init_at(), but if %ENOENT, then set - * @out_exists to %FALSE, and return successfully. - */ -static gboolean -dfd_iter_init_allow_noent (int dfd, - const char *path, - GLnxDirFdIterator *dfd_iter, - gboolean *out_exists, - GError **error) -{ - glnx_fd_close int fd = glnx_opendirat_with_errno (dfd, path, TRUE); - if (fd < 0) - { - if (errno != ENOENT) - return glnx_throw_errno (error); - *out_exists = FALSE; - return TRUE; - } - if (!glnx_dirfd_iterator_init_take_fd (fd, dfd_iter, error)) - return FALSE; - fd = -1; - *out_exists = TRUE; - return TRUE; -} - /* @deploydir_dfd: Directory FD for ostree/deploy * @osname: Target osname * @inout_deployments: All deployments in this subdir will be appended to this array @@ -64,7 +39,7 @@ _ostree_sysroot_list_deployment_dirs_for_os (int deploydir_dfd, g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; gboolean exists; const char *osdeploy_path = glnx_strjoina (osname, "/deploy"); - if (!dfd_iter_init_allow_noent (deploydir_dfd, osdeploy_path, &dfd_iter, &exists, error)) + if (!ot_dfd_iter_init_allow_noent (deploydir_dfd, osdeploy_path, &dfd_iter, &exists, error)) return FALSE; if (!exists) return TRUE; @@ -106,7 +81,7 @@ list_all_deployment_directories (OstreeSysroot *self, g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; gboolean exists; - if (!dfd_iter_init_allow_noent (self->sysroot_fd, "ostree/deploy", &dfd_iter, &exists, error)) + if (!ot_dfd_iter_init_allow_noent (self->sysroot_fd, "ostree/deploy", &dfd_iter, &exists, error)) return FALSE; if (!exists) return TRUE; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 510f2fb2..e47214c5 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -421,23 +421,17 @@ _ostree_sysroot_read_boot_loader_configs (OstreeSysroot *self, g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); g_autofree char *entries_path = g_strdup_printf ("boot/loader.%d/entries", bootversion); - /* Temporary owned by iterator */ - int fd = glnx_opendirat_with_errno (self->sysroot_fd, entries_path, TRUE); - if (fd == -1) - { - if (errno != ENOENT) - return glnx_throw_errno (error); - else - { - /* Note early return */ - *out_loader_configs = g_steal_pointer (&ret_loader_configs); - return TRUE; - } - } - + gboolean entries_exists; g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - if (!glnx_dirfd_iterator_init_take_fd (fd, &dfd_iter, error)) + if (!ot_dfd_iter_init_allow_noent (self->sysroot_fd, entries_path, + &dfd_iter, &entries_exists, error)) return FALSE; + if (!entries_exists) + { + /* Note early return */ + *out_loader_configs = g_steal_pointer (&ret_loader_configs); + return TRUE; + } while (TRUE) { diff --git a/src/libotutil/ot-fs-utils.c b/src/libotutil/ot-fs-utils.c index 8ba2cffb..529077fb 100644 --- a/src/libotutil/ot-fs-utils.c +++ b/src/libotutil/ot-fs-utils.c @@ -172,6 +172,31 @@ ot_openat_ignore_enoent (int dfd, return ret; } +/* Like glnx_dirfd_iterator_init_at(), but if %ENOENT, then set + * @out_exists to %FALSE, and return successfully. + */ +gboolean +ot_dfd_iter_init_allow_noent (int dfd, + const char *path, + GLnxDirFdIterator *dfd_iter, + gboolean *out_exists, + GError **error) +{ + glnx_fd_close int fd = glnx_opendirat_with_errno (dfd, path, TRUE); + if (fd < 0) + { + if (errno != ENOENT) + return glnx_throw_errno_prefix (error, "opendirat"); + *out_exists = FALSE; + return TRUE; + } + if (!glnx_dirfd_iterator_init_take_fd (fd, dfd_iter, error)) + return FALSE; + fd = -1; + *out_exists = TRUE; + return TRUE; +} + GBytes * ot_file_mapat_bytes (int dfd, const char *path, diff --git a/src/libotutil/ot-fs-utils.h b/src/libotutil/ot-fs-utils.h index c7770682..14df8acb 100644 --- a/src/libotutil/ot-fs-utils.h +++ b/src/libotutil/ot-fs-utils.h @@ -21,6 +21,7 @@ #pragma once #include "ot-unix-utils.h" +#include "libglnx.h" G_BEGIN_DECLS @@ -53,6 +54,12 @@ gboolean ot_openat_ignore_enoent (int dfd, int *out_fd, GError **error); +gboolean ot_dfd_iter_init_allow_noent (int dfd, + const char *path, + GLnxDirFdIterator *dfd_iter, + gboolean *out_exists, + GError **error); + GBytes *ot_file_mapat_bytes (int dfd, const char *path, GError **error); From 3ef287070089411350252cb0895d4f2cb8a861fa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 11:30:35 -0400 Subject: [PATCH 90/94] lib/upgrader: Port to new code style No surprises here. Prep for future work. Closes: #864 Approved by: jlebon --- src/libostree/ostree-sysroot-upgrader.c | 132 ++++++++---------------- 1 file changed, 41 insertions(+), 91 deletions(-) diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 11d9706c..9816b3d6 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -22,6 +22,7 @@ #include "otutil.h" +#include "ostree.h" #include "ostree-sysroot-upgrader.h" /** @@ -50,7 +51,7 @@ struct OstreeSysrootUpgrader { char *override_csum; char *new_revision; -}; +}; enum { PROP_0, @@ -70,7 +71,6 @@ parse_refspec (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autofree char *origin_refspec = NULL; g_autofree char *unconfigured_state = NULL; g_autofree char *csum = NULL; @@ -82,37 +82,28 @@ parse_refspec (OstreeSysrootUpgrader *self, */ unconfigured_state = g_key_file_get_string (self->origin, "origin", "unconfigured-state", NULL); if (unconfigured_state) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "origin unconfigured-state: %s", unconfigured_state); - goto out; - } + return glnx_throw (error, "origin unconfigured-state: %s", unconfigured_state); } origin_refspec = g_key_file_get_string (self->origin, "origin", "refspec", NULL); if (!origin_refspec) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No origin/refspec in current deployment origin; cannot upgrade via ostree"); - goto out; - } + return glnx_throw (error, "No origin/refspec in current deployment origin; cannot upgrade via ostree"); + g_clear_pointer (&self->origin_remote, g_free); g_clear_pointer (&self->origin_ref, g_free); if (!ostree_parse_refspec (origin_refspec, - &self->origin_remote, + &self->origin_remote, &self->origin_ref, error)) - goto out; + return FALSE; csum = g_key_file_get_string (self->origin, "origin", "override-commit", NULL); if (csum != NULL && !ostree_validate_checksum_string (csum, error)) - goto out; + return FALSE; g_clear_pointer (&self->override_csum, g_free); self->override_csum = g_steal_pointer (&csum); - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -120,17 +111,12 @@ ostree_sysroot_upgrader_initable_init (GInitable *initable, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; OstreeSysrootUpgrader *self = (OstreeSysrootUpgrader*)initable; OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (self->sysroot); if (booted_deployment == NULL && self->osname == NULL) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not currently booted into an OSTree system and no OS specified"); - goto out; - } + return glnx_throw (error, "Not currently booted into an OSTree system and no OS specified"); if (self->osname == NULL) { @@ -138,37 +124,23 @@ ostree_sysroot_upgrader_initable_init (GInitable *initable, self->osname = g_strdup (ostree_deployment_get_osname (booted_deployment)); } else if (self->osname[0] == '\0') - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid empty osname"); - goto out; - } + return glnx_throw (error, "Invalid empty osname"); - self->merge_deployment = ostree_sysroot_get_merge_deployment (self->sysroot, self->osname); + self->merge_deployment = ostree_sysroot_get_merge_deployment (self->sysroot, self->osname); if (self->merge_deployment == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No previous deployment for OS '%s'", self->osname); - goto out; - } + return glnx_throw (error, "No previous deployment for OS '%s'", self->osname); self->origin = ostree_deployment_get_origin (self->merge_deployment); if (!self->origin) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No origin known for deployment %s.%d", - ostree_deployment_get_csum (self->merge_deployment), - ostree_deployment_get_deployserial (self->merge_deployment)); - goto out; - } + return glnx_throw (error, "No origin known for deployment %s.%d", + ostree_deployment_get_csum (self->merge_deployment), + ostree_deployment_get_deployserial (self->merge_deployment)); g_key_file_ref (self->origin); if (!parse_refspec (self, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } static void @@ -401,19 +373,15 @@ ostree_sysroot_upgrader_set_origin (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_clear_pointer (&self->origin, g_key_file_unref); if (origin) { self->origin = g_key_file_ref (origin); if (!parse_refspec (self, cancellable, error)) - goto out; + return FALSE; } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -447,21 +415,19 @@ ostree_sysroot_upgrader_check_timestamps (OstreeRepo *repo, const char *to_rev, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) old_commit = NULL; - g_autoptr(GVariant) new_commit = NULL; - if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, from_rev, &old_commit, error)) - goto out; - + return FALSE; + + g_autoptr(GVariant) new_commit = NULL; if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, to_rev, &new_commit, error)) - goto out; + return FALSE; if (ostree_commit_get_timestamp (old_commit) > ostree_commit_get_timestamp (new_commit)) { @@ -471,31 +437,22 @@ ostree_sysroot_upgrader_check_timestamps (OstreeRepo *repo, g_autofree char *new_ts_str = NULL; if (old_ts == NULL || new_ts == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Upgrade target revision '%s' timestamp (%" G_GINT64_FORMAT ") or current revision '%s' timestamp (%" G_GINT64_FORMAT ") is invalid", - to_rev, ostree_commit_get_timestamp (new_commit), - from_rev, ostree_commit_get_timestamp (old_commit)); - goto out; - } + return glnx_throw (error, "Upgrade target revision '%s' timestamp (%" G_GINT64_FORMAT ") or current revision '%s' timestamp (%" G_GINT64_FORMAT ") is invalid", + to_rev, ostree_commit_get_timestamp (new_commit), + from_rev, ostree_commit_get_timestamp (old_commit)); old_ts_str = g_date_time_format (old_ts, "%c"); new_ts_str = g_date_time_format (new_ts, "%c"); g_date_time_unref (old_ts); g_date_time_unref (new_ts); - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Upgrade target revision '%s' with timestamp '%s' is chronologically older than current revision '%s' with timestamp '%s'; use --allow-downgrade to permit", - to_rev, new_ts_str, from_rev, old_ts_str); - goto out; + return glnx_throw (error, "Upgrade target revision '%s' with timestamp '%s' is chronologically older than current revision '%s' with timestamp '%s'; use --allow-downgrade to permit", + to_rev, new_ts_str, from_rev, old_ts_str); } - ret = TRUE; - out: - return ret; + return TRUE; } - /** * ostree_sysroot_upgrader_pull: * @self: Upgrader @@ -551,8 +508,7 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - glnx_unref_object OstreeRepo *repo = NULL; + g_autoptr(OstreeRepo) repo = NULL; char *refs_to_fetch[] = { NULL, NULL }; const char *from_revision = NULL; g_autofree char *origin_refspec = NULL; @@ -563,7 +519,7 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, refs_to_fetch[0] = self->origin_ref; if (!ostree_sysroot_get_repo (self->sysroot, &repo, cancellable, error)) - goto out; + return FALSE; if (self->origin_remote) origin_refspec = g_strconcat (self->origin_remote, ":", self->origin_ref, NULL); @@ -579,7 +535,7 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch, flags, progress, cancellable, error)) - goto out; + return FALSE; if (progress) ostree_async_progress_finish (progress); @@ -593,7 +549,7 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, self->override_csum, cancellable, error)) - goto out; + return FALSE; self->new_revision = g_strdup (self->override_csum); } @@ -601,7 +557,7 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, { if (!ostree_repo_resolve_rev (repo, origin_refspec, FALSE, &self->new_revision, error)) - goto out; + return FALSE; } @@ -620,13 +576,11 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, if (!ostree_sysroot_upgrader_check_timestamps (repo, from_revision, self->new_revision, error)) - goto out; + return FALSE; } } - ret = TRUE; - out: - return ret; + return TRUE; } /** @@ -643,9 +597,7 @@ ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - glnx_unref_object OstreeDeployment *new_deployment = NULL; - + g_autoptr(OstreeDeployment) new_deployment = NULL; if (!ostree_sysroot_deploy_tree (self->sysroot, self->osname, self->new_revision, self->origin, @@ -653,18 +605,16 @@ ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader *self, NULL, &new_deployment, cancellable, error)) - goto out; + return FALSE; if (!ostree_sysroot_simple_write_deployment (self->sysroot, self->osname, new_deployment, self->merge_deployment, 0, cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - out: - return ret; + return TRUE; } GType From 28e30712564278807bb2fb71ea08bc1d127b83f0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 21:46:22 -0400 Subject: [PATCH 91/94] build: Use cd $(srcdir) instead of `git -C` Since the version in CentOS is too old, and we get a spam of warnings, plus things like detecting the git repo break. Fixes: 50f73cbac35be97fd5895531e295d05dabaa8ed9 Closes: #868 Approved by: jlebon --- Makefile.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index be505522..53b505e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,7 +19,7 @@ include Makefile-decls.am shortened_sysconfdir = $$(echo "$(sysconfdir)" | sed -e 's|^$(prefix)||' -e 's|^/||') -OSTREE_GITREV=$(shell if command -v git >/dev/null 2>&1 && test -d $(srcdir)/.git; then git -C $(srcdir) describe --abbrev=42 --tags --always HEAD; fi) +OSTREE_GITREV=$(shell cd $(srcdir) && if command -v git >/dev/null 2>&1 && test -d .git; then git describe --abbrev=42 --tags --always HEAD; fi) ACLOCAL_AMFLAGS = -I buildutil -I libglnx ${ACLOCAL_FLAGS} AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ @@ -113,11 +113,11 @@ include Makefile-boot.am include Makefile-man.am release-tag: - git -C $(srcdir) tag -m "Release $(VERSION)" v$(VERSION) + cd $(srcdir) && git $(srcdir) tag -m "Release $(VERSION)" v$(VERSION) embed_dependency=tar -C $(srcdir) --append --exclude='.git/*' --transform="s,^embedded-dependencies/,ostree-embeddeps-$${GITVERSION}/embedded-dependencies/," --file=$${TARFILE_TMP} -git_version_rpm = $$(git -C $(srcdir) describe | sed -e 's,-,\.,g' -e 's,^v,,') +git_version_rpm = $$(cd $(srcdir) && git describe | sed -e 's,-,\.,g' -e 's,^v,,') release-tarball-embedded: set -x; \ From 1a8f2f0769fd97c59b3d5e25f6be62c586ca6c06 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 21:44:22 -0400 Subject: [PATCH 92/94] switchroot/generator: Add var.mount to local-fs.target.requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unbreaks mounting in CentOS. Newer systemd in Fedora pulls didn't need this, I think due to `RequiresMountsFor=`. Anyways, this is what the fstab generator does, and it's clearly right ✓. Closes: https://github.com/ostreedev/ostree/issues/867 Closes: #869 Approved by: jlebon --- src/libostree/ostree-impl-system-generator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libostree/ostree-impl-system-generator.c b/src/libostree/ostree-impl-system-generator.c index 60df145a..72f52bc5 100644 --- a/src/libostree/ostree-impl-system-generator.c +++ b/src/libostree/ostree-impl-system-generator.c @@ -212,6 +212,16 @@ _ostree_impl_system_generator (const char *ostree_cmdline, "var.mount", error)) return FALSE; + /* And ensure it's required; newer systemd will auto-inject fs dependencies + * via RequiresMountsFor and the like, but on older versions (e.g. CentOS) we + * need this. It's what the fstab generator does. And my mother always said, + * listen to the fstab generator. + */ + if (!glnx_shutil_mkdir_p_at (normal_dir_dfd, "local-fs.target.requires", 0755, cancellable, error)) + return FALSE; + if (symlinkat ("../var.mount", normal_dir_dfd, "local-fs.target.requires/var.mount") < 0) + return glnx_throw_errno_prefix (error, "symlinkat"); + return TRUE; #else return glnx_throw (error, "Not implemented"); From 1470ff58b0accaaec5093eadc0e4eb0bc2538af4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 17 May 2017 09:55:34 -0400 Subject: [PATCH 93/94] lib/pull: Port some functions to new code style Porting a lot of this file would be hard since in many cases we do processing in the `out:` section, so let's do what we can. Closes: #870 Approved by: jlebon --- src/libostree/ostree-repo-pull.c | 137 ++++++++++--------------------- 1 file changed, 42 insertions(+), 95 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7929b91a..f1dae995 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -437,32 +437,22 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; g_autoptr(GBytes) bytes = NULL; - g_autofree char *ret_contents = NULL; - gsize len; - if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, TRUE, FALSE, &bytes, OSTREE_MAX_METADATA_SIZE, cancellable, error)) - goto out; + return FALSE; - ret_contents = g_bytes_unref_to_data (bytes, &len); - bytes = NULL; + gsize len; + g_autofree char *ret_contents = g_bytes_unref_to_data (g_steal_pointer (&bytes), &len); if (!g_utf8_validate (ret_contents, -1, NULL)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid UTF-8"); - goto out; - } + return glnx_throw (error, "Invalid UTF-8"); - ret = TRUE; ot_transfer_out_value (out_contents, &ret_contents); - out: - return ret; + return TRUE; } static gboolean @@ -485,16 +475,11 @@ write_commitpartial_for (OtPullData *pull_data, GError **error) { g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum); - glnx_fd_close int fd = -1; - - fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644); + glnx_fd_close int fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644); if (fd == -1) { if (errno != EEXIST) - { - glnx_set_error_from_errno (error); - return FALSE; - } + return glnx_throw_errno_prefix (error, "open(%s)", commitpartial_path); } return TRUE; } @@ -555,15 +540,12 @@ pull_matches_subdir (OtPullData *pull_data, const char *basename, gboolean basename_is_dir) { - int i; - g_autofree char *file = NULL; - if (pull_data->dirs == NULL) return TRUE; - file = g_strconcat (path, basename, NULL); + g_autofree char *file = g_strconcat (path, basename, NULL); - for (i = 0; i < pull_data->dirs->len; i++) + for (guint i = 0; i < pull_data->dirs->len; i++) { const char *pull_dir = g_ptr_array_index (pull_data->dirs, i); if (matches_pull_dir (file, pull_dir, basename_is_dir)) @@ -581,30 +563,18 @@ scan_dirtree_object (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - int i, n; - g_autoptr(GVariant) tree = NULL; - g_autoptr(GVariant) files_variant = NULL; - g_autoptr(GVariant) dirs_variant = NULL; - const char *dirname = NULL; - if (recursion_depth > OSTREE_MAX_RECURSION) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Exceeded maximum recursion"); - goto out; - } + return glnx_throw (error, "Exceeded maximum recursion"); + g_autoptr(GVariant) tree = NULL; if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_DIR_TREE, checksum, &tree, error)) - goto out; + return FALSE; /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ - files_variant = g_variant_get_child_value (tree, 0); - dirs_variant = g_variant_get_child_value (tree, 1); - - n = g_variant_n_children (files_variant); - for (i = 0; i < n; i++) + g_autoptr(GVariant) files_variant = g_variant_get_child_value (tree, 0); + const guint n = g_variant_n_children (files_variant); + for (guint i = 0; i < n; i++) { const char *filename; gboolean file_is_stored; @@ -614,7 +584,7 @@ scan_dirtree_object (OtPullData *pull_data, g_variant_get_child (files_variant, i, "(&s@ay)", &filename, &csum); if (!ot_util_filename_validate (filename, error)) - goto out; + return FALSE; /* Skip files if we're traversing a request only directory, unless it exactly * matches the path */ @@ -625,14 +595,14 @@ scan_dirtree_object (OtPullData *pull_data, if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, &file_is_stored, cancellable, error)) - goto out; + return FALSE; if (!file_is_stored && pull_data->remote_repo_local) { if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, OSTREE_OBJECT_TYPE_FILE, file_checksum, !pull_data->is_untrusted, cancellable, error)) - goto out; + return FALSE; } else if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum)) { @@ -642,44 +612,38 @@ scan_dirtree_object (OtPullData *pull_data, } } - n = g_variant_n_children (dirs_variant); - - for (i = 0; i < n; i++) + g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1); + const guint m = g_variant_n_children (dirs_variant); + for (guint i = 0; i < m; i++) { + const char *dirname = NULL; g_autoptr(GVariant) tree_csum = NULL; g_autoptr(GVariant) meta_csum = NULL; - const guchar *tree_csum_bytes; - const guchar *meta_csum_bytes; - g_autofree char *subpath = NULL; - g_variant_get_child (dirs_variant, i, "(&s@ay@ay)", &dirname, &tree_csum, &meta_csum); if (!ot_util_filename_validate (dirname, error)) - goto out; + return FALSE; if (!pull_matches_subdir (pull_data, path, dirname, TRUE)) continue; - tree_csum_bytes = ostree_checksum_bytes_peek_validate (tree_csum, error); + const guchar *tree_csum_bytes = ostree_checksum_bytes_peek_validate (tree_csum, error); if (tree_csum_bytes == NULL) - goto out; + return FALSE; - meta_csum_bytes = ostree_checksum_bytes_peek_validate (meta_csum, error); + const guchar *meta_csum_bytes = ostree_checksum_bytes_peek_validate (meta_csum, error); if (meta_csum_bytes == NULL) - goto out; - - subpath = g_strconcat (path, dirname, "/", NULL); + return FALSE; + g_autofree char *subpath = g_strconcat (path, dirname, "/", NULL); queue_scan_one_metadata_object_c (pull_data, tree_csum_bytes, OSTREE_OBJECT_TYPE_DIR_TREE, subpath, recursion_depth + 1); queue_scan_one_metadata_object_c (pull_data, meta_csum_bytes, OSTREE_OBJECT_TYPE_DIR_META, subpath, recursion_depth + 1); } - ret = TRUE; - out: - return ret; + return TRUE; } static gboolean @@ -689,27 +653,21 @@ fetch_ref_contents (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; + g_autofree char *filename = g_build_filename ("refs", "heads", ref, NULL); g_autofree char *ret_contents = NULL; - g_autofree char *filename = NULL; - - filename = g_build_filename ("refs", "heads", ref, NULL); - if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, pull_data->meta_mirrorlist, filename, &ret_contents, cancellable, error)) - goto out; + return FALSE; g_strchomp (ret_contents); if (!ostree_validate_checksum_string (ret_contents, error)) - goto out; + return FALSE; - ret = TRUE; ot_transfer_out_value (out_contents, &ret_contents); - out: - return ret; + return TRUE; } static gboolean @@ -719,34 +677,23 @@ lookup_commit_checksum_from_summary (OtPullData *pull_data, gsize *out_size, GError **error) { - gboolean ret = FALSE; g_autoptr(GVariant) refs = g_variant_get_child_value (pull_data->summary, 0); - g_autoptr(GVariant) refdata = NULL; - g_autoptr(GVariant) reftargetdata = NULL; + int i; + if (!ot_variant_bsearch_str (refs, ref, &i)) + return glnx_throw (error, "No such branch '%s' in repository summary", ref); + + g_autoptr(GVariant) refdata = g_variant_get_child_value (refs, i); + g_autoptr(GVariant) reftargetdata = g_variant_get_child_value (refdata, 1); guint64 commit_size; g_autoptr(GVariant) commit_csum_v = NULL; - int i; - - if (!ot_variant_bsearch_str (refs, ref, &i)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No such branch '%s' in repository summary", - ref); - goto out; - } - - refdata = g_variant_get_child_value (refs, i); - reftargetdata = g_variant_get_child_value (refdata, 1); g_variant_get (reftargetdata, "(t@ay@a{sv})", &commit_size, &commit_csum_v, NULL); if (!ostree_validate_structureof_csum_v (commit_csum_v, error)) - goto out; + return FALSE; - ret = TRUE; *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v); *out_size = commit_size; - out: - return ret; + return TRUE; } static void @@ -772,7 +719,7 @@ content_fetch_on_write_complete (GObject *object, g_autofree char *checksum = NULL; g_autofree char *checksum_obj = NULL; - if (!ostree_repo_write_content_finish ((OstreeRepo*)object, result, + if (!ostree_repo_write_content_finish ((OstreeRepo*)object, result, &csum, error)) goto out; From 88792f0f22a2560374d6e7e8f37e1f670f335b3f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 May 2017 12:02:25 -0400 Subject: [PATCH 94/94] Release 2017.6 There's already a lot queued. In particular this brings some API additions that rpm-ostree depends on. Closes: #865 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree.sym | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 06d39126..a6abee85 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ AC_PREREQ([2.63]) dnl If incrementing the version here, remember to update libostree.sym too m4_define([year_version], [2017]) -m4_define([release_version], [5]) +m4_define([release_version], [6]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index df74de53..e2a7c3d9 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -389,11 +389,6 @@ global: ostree_sysroot_write_deployments_with_options; } LIBOSTREE_2017.3; -/* NOTE NOTE NOTE - * Versions above here are released. Only add symbols below this line. - * NOTE NOTE NOTE - */ - LIBOSTREE_2017.6 { global: ostree_async_progress_get; @@ -402,6 +397,18 @@ global: ostree_async_progress_set_variant; } LIBOSTREE_2017.4; +/* NOTE NOTE NOTE + * Versions above here are released. Only add symbols below this line. + * NOTE NOTE NOTE + */ + +/* +LIBOSTREE_2017.$NEWVERSION { +global: + someostree_symbol_deleteme; +} LIBOSTREE_2017.6; +*/ + /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION