From 620a90ebfaf98f779b77cd8f97da1d06e146b832 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 5 Jul 2017 16:41:38 -0400 Subject: [PATCH] lib/pull: Avoid journaling 404s for optional content Currently in Fedora we don't sign summaries, and every use of `rpm-ostree` would emit to the journal an error when we failed to fetch it. Fix this by having `OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT` tell the fetcher not to journal 404 errors. While fixing this, we had a mix of two booleans vs the flags; fix things so we consistently use the flags in the fetcher and pull code. Closes: #1004 Approved by: mbarnes --- src/libostree/ostree-fetcher-curl.c | 7 +++++-- src/libostree/ostree-fetcher-soup.c | 7 +++++-- src/libostree/ostree-fetcher-util.c | 13 +++++-------- src/libostree/ostree-fetcher-util.h | 6 ++---- src/libostree/ostree-fetcher.h | 10 ++++++---- src/libostree/ostree-metalink.c | 16 ++++------------ src/libostree/ostree-repo-pull.c | 18 ++++++++++-------- 7 files changed, 37 insertions(+), 40 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 77844ec7..ae8bea73 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -353,7 +353,9 @@ check_multi_info (OstreeFetcher *fetcher) g_autofree char *msg = g_strdup_printf ("Server returned HTTP %lu", response); g_task_return_new_error (task, G_IO_ERROR, giocode, "%s", msg); - if (req->fetcher->remote_name) + if (req->fetcher->remote_name && + !((req->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + giocode == G_IO_ERROR_NOT_FOUND)) _ostree_fetcher_journal_failure (req->fetcher->remote_name, eff_url, msg); @@ -859,13 +861,14 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, + OstreeFetcherRequestFlags flags, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE, max_size, priority, cancellable, callback, user_data); } diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index b877d27c..16fda0a3 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1128,7 +1128,9 @@ on_request_sent (GObject *object, g_prefix_error (&local_error, "All %u mirrors failed. Last error was: ", pending->mirrorlist->len); - if (pending->thread_closure->remote_name) + if (pending->thread_closure->remote_name && + !((pending->flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) > 0 && + code == G_IO_ERROR_NOT_FOUND)) _ostree_fetcher_journal_failure (pending->thread_closure->remote_name, uristring, local_error->message); @@ -1238,13 +1240,14 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, + OstreeFetcherRequestFlags flags, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE, max_size, priority, cancellable, callback, user_data); } diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index 408b8bcb..7a6d94cc 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -55,8 +55,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, GPtrArray *mirrorlist, const char *filename, - gboolean add_nul, - gboolean allow_noent, + OstreeFetcherRequestFlags flags, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -79,7 +78,7 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, data.error = error; _ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename, - add_nul ? OSTREE_FETCHER_REQUEST_NUL_TERMINATION : 0, + flags, max_size, OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable, fetch_uri_sync_on_complete, &data); while (!data.done) @@ -87,7 +86,7 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, if (!data.result_buf) { - if (allow_noent) + if (flags & OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT) { if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { @@ -112,8 +111,7 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, - gboolean add_nul, - gboolean allow_noent, + OstreeFetcherRequestFlags flags, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -121,8 +119,7 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, { g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); g_ptr_array_add (mirrorlist, uri); /* no transfer */ - return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, - add_nul, allow_noent, + return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags, out_contents, max_size, cancellable, error); } diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h index fe0921cd..c099a872 100644 --- a/src/libostree/ostree-fetcher-util.h +++ b/src/libostree/ostree-fetcher-util.h @@ -29,8 +29,7 @@ G_BEGIN_DECLS gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, GPtrArray *mirrorlist, const char *filename, - gboolean add_nul, - gboolean allow_noent, + OstreeFetcherRequestFlags flags, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, @@ -38,8 +37,7 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, - gboolean add_nul, - gboolean allow_noent, + OstreeFetcherRequestFlags flags, GBytes **out_contents, guint64 max_size, GCancellable *cancellable, diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 8ec5f209..2dbace78 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -54,6 +54,11 @@ typedef enum { OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1) } OstreeFetcherConfigFlags; +typedef enum { + OSTREE_FETCHER_REQUEST_NUL_TERMINATION = (1 << 0), + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT = (1 << 1) +} OstreeFetcherRequestFlags; + void _ostree_fetcher_uri_free (OstreeFetcherURI *uri); G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcherURI, _ostree_fetcher_uri_free) @@ -111,6 +116,7 @@ guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self); void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, + OstreeFetcherRequestFlags flags, guint64 max_size, int priority, GCancellable *cancellable, @@ -122,10 +128,6 @@ gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, char **out_filename, GError **error); -typedef enum { - OSTREE_FETCHER_REQUEST_NUL_TERMINATION = (1 << 0) -} OstreeFetcherRequestFlags; - void _ostree_fetcher_request_to_membuf (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c index 6afae59e..0b1b14fb 100644 --- a/src/libostree/ostree-metalink.c +++ b/src/libostree/ostree-metalink.c @@ -431,10 +431,7 @@ try_one_url (OstreeMetalinkRequest *self, gssize n_bytes; if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher, - uri, - FALSE, - FALSE, - &bytes, + uri, 0, &bytes, self->metalink->max_size, self->cancellable, error)) @@ -614,14 +611,9 @@ _ostree_metalink_request_sync (OstreeMetalink *self, request.urls = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL); - if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, - self->uri, - FALSE, - FALSE, - &contents, - self->max_size, - cancellable, - error)) + if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0, + &contents, self->max_size, + cancellable, error)) goto out; data = g_bytes_get_data (contents, &len); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5e0ae6ec..34ea7f9e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -470,7 +470,7 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, { g_autoptr(GBytes) bytes = NULL; if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, - filename, TRUE, FALSE, + filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION, &bytes, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -1864,6 +1864,7 @@ start_fetch (OtPullData *pull_data, else pull_data->n_outstanding_content_fetches++; + OstreeFetcherRequestFlags flags = 0; /* Override the path if we're trying to fetch the .commitmeta file first */ if (fetch->is_detached_meta) { @@ -1871,6 +1872,7 @@ start_fetch (OtPullData *pull_data, _ostree_loose_path (buf, expected_checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode); obj_subpath = g_build_filename ("objects", buf, NULL); mirrorlist = pull_data->meta_mirrorlist; + flags |= OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT; } else { @@ -1890,7 +1892,7 @@ start_fetch (OtPullData *pull_data, expected_max_size = 0; _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist, - obj_subpath, expected_max_size, + obj_subpath, flags, expected_max_size, is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY : OSTREE_REPO_PULL_CONTENT_PRIORITY, pull_data->cancellable, @@ -1995,7 +1997,7 @@ start_fetch_deltapart (OtPullData *pull_data, g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, pull_data->content_mirrorlist, - deltapart_path, fetch->size, + deltapart_path, 0, fetch->size, OSTREE_FETCHER_DEFAULT_PRIORITY, pull_data->cancellable, static_deltapart_fetch_on_complete, @@ -2677,7 +2679,8 @@ _ostree_preload_metadata_file (OstreeRepo *self, else { ret = _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, - filename, FALSE, TRUE, + filename, + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, out_bytes, OSTREE_MAX_METADATA_SIZE, cancellable, error); @@ -3514,7 +3517,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, - "summary.sig", FALSE, TRUE, + "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, &bytes_sig, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -3538,7 +3541,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, - "summary", FALSE, TRUE, + "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, &bytes_summary, OSTREE_MAX_METADATA_SIZE, cancellable, error)) @@ -4776,8 +4779,7 @@ find_remotes_cb (GObject *obj, if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, commit_filename, - FALSE, /* don’t add trailing nul */ - TRUE, /* return NULL on ENOENT */ + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, &commit_bytes, 0, /* no maximum size */ cancellable,