From f5da67d78a0e64117169762d102a706b941c5945 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 16 Jul 2020 16:16:37 +0100 Subject: [PATCH] pull: Add summary-{,sig-}bytes options to ostree_repo_pull() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These allow the `summary` and `summary.sig` files to be cached at a higher layer (for example, flatpak) between related pull operations (for example, within a single flatpak transaction). This avoids re-downloading `summary.sig` multiple times throughout a transaction, which increases the transaction’s latency and introduces the possibility for inconsistency between parts of the transaction if the server changes its `summary` file part-way through. In particular, this should speed up flatpak transactions on machines with high latency network connections, where network round trips have a high impact on the latency of an overall operation. Signed-off-by: Philip Withnall --- src/libostree/ostree-repo-pull.c | 43 ++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 18fe9d27..894e4b1f 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3311,6 +3311,14 @@ initiate_request (OtPullData *pull_data, * not being pulled will be ignored and any ref without a keyring remote * will be verified with the keyring of the remote being pulled from. * Since: 2019.2 + * * `summary-bytes` (`ay'): Contents of the `summary` file to use. If this is + * specified, `summary-sig-bytes` must also be specified. This is + * useful if doing multiple pull operations in a transaction, using + * ostree_repo_remote_fetch_summary_with_options() beforehand to download + * the `summary` and `summary.sig` once for the entire transaction. If not + * specified, the `summary` will be downloaded from the remote. Since: 2020.5 + * * `summary-sig-bytes` (`ay`): Contents of the `summary.sig` file. If this + * is specified, `summary-bytes` must also be specified. Since: 2020.5 */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3356,6 +3364,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, int i; g_autofree char **opt_localcache_repos = NULL; g_autoptr(GVariantIter) ref_keyring_map_iter = NULL; + g_autoptr(GVariant) summary_bytes_v = NULL; + g_autoptr(GVariant) summary_sig_bytes_v = NULL; /* If refs or collection-refs has exactly one value, this will point to that * value, otherwise NULL. Used for logging. */ @@ -3402,6 +3412,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_variant_lookup (options, "n-network-retries", "u", &pull_data->n_network_retries); opt_ref_keyring_map_set = g_variant_lookup (options, "ref-keyring-map", "a(sss)", &ref_keyring_map_iter); + (void) g_variant_lookup (options, "summary-bytes", "@ay", &summary_bytes_v); + (void) g_variant_lookup (options, "summary-sig-bytes", "@ay", &summary_sig_bytes_v); if (pull_data->remote_refspec_name != NULL) pull_data->remote_name = g_strdup (pull_data->remote_refspec_name); @@ -3439,6 +3451,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, */ g_return_val_if_fail (!pull_data->dry_run || pull_data->require_static_deltas, FALSE); + /* summary-bytes and summary-sig-bytes must both be specified, or neither be + * specified, so we know they’re consistent */ + g_return_val_if_fail ((summary_bytes_v == NULL) == (summary_sig_bytes_v == NULL), FALSE); + pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; /* See our processing of OSTREE_REPO_PULL_FLAGS_UNTRUSTED below */ @@ -3651,6 +3667,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!metalink_uri) goto out; + /* FIXME: Use summary_bytes_v/summary_sig_bytes_v to avoid unnecessary + * re-downloads here. Would require additional support for caching the + * metalink file or mirror list. */ + metalink = _ostree_metalink_new (pull_data->fetcher, "summary", OSTREE_MAX_METADATA_SIZE, metalink_uri, pull_data->n_network_retries); @@ -3841,7 +3861,25 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GVariant) additional_metadata = NULL; gboolean summary_from_cache = FALSE; - if (!pull_data->summary_data_sig) + if (summary_sig_bytes_v) + { + /* Must both be specified */ + g_assert (summary_bytes_v); + + bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v); + bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v); + + if (!bytes_sig || !bytes_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "summary-bytes or summary-sig-bytes set to invalid value"); + goto out; + } + + g_debug ("Loaded %s summary from options", remote_name_or_baseurl); + } + + if (!bytes_sig) { if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, @@ -3854,6 +3892,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } if (bytes_sig && + !bytes_summary && !pull_data->remote_repo_local && !_ostree_repo_load_cache_summary_if_same_sig (self, remote_name_or_baseurl, @@ -3863,7 +3902,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, error)) goto out; - if (bytes_summary) + if (bytes_summary && !summary_bytes_v) { g_debug ("Loaded %s summary from cache", remote_name_or_baseurl); summary_from_cache = TRUE;