From 157d878ce1a500a2b96d079e2259dbdc30419342 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 23 Aug 2016 15:55:15 -0400 Subject: [PATCH] pull: add mirrorlist support This commit adds mirrorlist support to the fetcher. Users can now prepend url or/and contenturl by mirrorlist= to interpret the link as a mirrorlist. If an object is not found, the fetcher will automatically try the next mirror in the order given in the list (assuming the order returned by the server is significant). Closes: #469 Approved by: cgwalters --- src/libostree/ostree-fetcher.c | 223 ++++++++++----- src/libostree/ostree-fetcher.h | 33 ++- src/libostree/ostree-repo-pull.c | 443 +++++++++++++++++++---------- src/ostree/ot-remote-builtin-add.c | 2 +- 4 files changed, 461 insertions(+), 240 deletions(-) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index bde6ed9a..09006b71 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -74,7 +74,9 @@ typedef struct { volatile int ref_count; ThreadClosure *thread_closure; - SoupURI *uri; + GPtrArray *mirrorlist; /* list of base URIs */ + char *filename; /* relative name to fetch or NULL */ + guint mirrorlist_idx; OstreeFetcherState state; @@ -204,7 +206,8 @@ pending_uri_unref (OstreeFetcherPendingURI *pending) g_clear_pointer (&pending->thread_closure, thread_closure_unref); - soup_uri_free (pending->uri); + g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref); + g_free (pending->filename); g_clear_object (&pending->request); g_clear_object (&pending->request_body); g_free (pending->out_tmpfile); @@ -353,6 +356,31 @@ session_thread_process_pending_queue (ThreadClosure *thread_closure) } } +static void +create_pending_soup_request (OstreeFetcherPendingURI *pending, + GError **error) +{ + g_autofree char *uristr = NULL; + SoupURI *next_mirror = NULL; + SoupURI *uri = NULL; + + g_assert (pending->mirrorlist); + g_assert (pending->mirrorlist_idx < pending->mirrorlist->len); + + next_mirror = g_ptr_array_index (pending->mirrorlist, + pending->mirrorlist_idx); + uristr = g_build_filename (soup_uri_get_path (next_mirror), + pending->filename /* may be NULL */, NULL); + uri = soup_uri_copy (next_mirror); + soup_uri_set_path (uri, uristr); + + g_clear_object (&pending->request); + + pending->request = soup_session_request_uri (pending->thread_closure->session, + uri, error); + soup_uri_free (uri); +} + static void session_thread_request_uri (ThreadClosure *thread_closure, gpointer data) @@ -365,10 +393,7 @@ session_thread_request_uri (ThreadClosure *thread_closure, pending = g_task_get_task_data (task); cancellable = g_task_get_cancellable (task); - pending->request = soup_session_request_uri (thread_closure->session, - pending->uri, - &local_error); - + create_pending_soup_request (pending, &local_error); if (local_error != NULL) { g_task_return_error (task, local_error); @@ -384,7 +409,8 @@ session_thread_request_uri (ThreadClosure *thread_closure, } else { - g_autofree char *uristring = soup_uri_to_string (pending->uri, FALSE); + g_autofree char *uristring + = soup_uri_to_string (soup_request_get_uri (pending->request), FALSE); g_autofree char *tmpfile = NULL; struct stat stbuf; gboolean exists; @@ -463,6 +489,8 @@ ostree_fetcher_session_thread (gpointer data) SOUP_SESSION_IDLE_TIMEOUT, 60, NULL); + /* XXX: Now that we have mirrorlist support, we could make this even smarter + * by spreading requests across mirrors. */ g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL); if (max_conns < 8) { @@ -856,7 +884,8 @@ on_stream_read (GObject *object, if (bytes_read > pending->max_size || (bytes_read + pending->current_size) > pending->max_size) { - g_autofree char *uristr = soup_uri_to_string (pending->uri, FALSE); + g_autofree char *uristr = + soup_uri_to_string (soup_request_get_uri (pending->request), FALSE); local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", uristr, pending->max_size); @@ -937,20 +966,43 @@ on_request_sent (GObject *object, } else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) { - GIOErrorEnum code; - switch (msg->status_code) + /* is there another mirror we can try? */ + if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len) { - case 404: - case 410: - code = G_IO_ERROR_NOT_FOUND; - break; - default: - code = G_IO_ERROR_FAILED; + pending->mirrorlist_idx++; + create_pending_soup_request (pending, &local_error); + if (local_error != NULL) + goto out; + + (void) g_input_stream_close (pending->request_body, NULL, NULL); + g_queue_insert_sorted (&pending->thread_closure->pending_queue, + g_object_ref (task), pending_task_compare, + NULL); + remove_pending_rerun_queue (pending); + } + else + { + GIOErrorEnum code; + switch (msg->status_code) + { + case 404: + case 410: + code = G_IO_ERROR_NOT_FOUND; + break; + default: + code = G_IO_ERROR_FAILED; + } + + local_error = g_error_new (G_IO_ERROR, code, + "Server returned status %u: %s", + msg->status_code, + soup_status_get_phrase (msg->status_code)); + + if (pending->mirrorlist->len > 1) + g_prefix_error (&local_error, + "All %u mirrors failed. Last error was: ", + pending->mirrorlist->len); } - local_error = g_error_new (G_IO_ERROR, code, - "Server returned status %u: %s", - msg->status_code, - soup_status_get_phrase (msg->status_code)); goto out; } } @@ -1013,27 +1065,30 @@ on_request_sent (GObject *object, } static void -ostree_fetcher_request_uri_internal (OstreeFetcher *self, - SoupURI *uri, - gboolean is_stream, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data, - gpointer source_tag) +ostree_fetcher_mirrored_request_internal (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + gboolean is_stream, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data, + gpointer source_tag) { g_autoptr(GTask) task = NULL; OstreeFetcherPendingURI *pending; g_return_if_fail (OSTREE_IS_FETCHER (self)); - g_return_if_fail (uri != NULL); + g_return_if_fail (mirrorlist != NULL); + g_return_if_fail (mirrorlist->len > 0); /* SoupRequest is created in session thread. */ pending = g_new0 (OstreeFetcherPendingURI, 1); pending->ref_count = 1; pending->thread_closure = thread_closure_ref (self->thread_closure); - pending->uri = soup_uri_copy (uri); + pending->mirrorlist = g_ptr_array_ref (mirrorlist); + pending->filename = g_strdup (filename); pending->max_size = max_size; pending->is_stream = is_stream; @@ -1051,53 +1106,57 @@ ostree_fetcher_request_uri_internal (OstreeFetcher *self, } void -_ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self, - SoupURI *uri, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data) +_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) { - ostree_fetcher_request_uri_internal (self, uri, FALSE, max_size, priority, cancellable, - callback, user_data, - _ostree_fetcher_request_uri_with_partial_async); + ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE, + max_size, priority, cancellable, + callback, user_data, + _ostree_fetcher_mirrored_request_with_partial_async); } char * -_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self, - GAsyncResult *result, - GError **error) +_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self, + GAsyncResult *result, + GError **error) { g_return_val_if_fail (g_task_is_valid (result, self), NULL); g_return_val_if_fail (g_async_result_is_tagged (result, - _ostree_fetcher_request_uri_with_partial_async), NULL); + _ostree_fetcher_mirrored_request_with_partial_async), NULL); return g_task_propagate_pointer (G_TASK (result), error); } static void -ostree_fetcher_stream_uri_async (OstreeFetcher *self, - SoupURI *uri, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data) +ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) { - ostree_fetcher_request_uri_internal (self, uri, TRUE, max_size, priority, cancellable, - callback, user_data, - ostree_fetcher_stream_uri_async); + ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE, + max_size, priority, cancellable, + callback, user_data, + ostree_fetcher_stream_mirrored_uri_async); } static GInputStream * -ostree_fetcher_stream_uri_finish (OstreeFetcher *self, - GAsyncResult *result, - GError **error) +ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher *self, + GAsyncResult *result, + GError **error) { g_return_val_if_fail (g_task_is_valid (result, self), NULL); g_return_val_if_fail (g_async_result_is_tagged (result, - ostree_fetcher_stream_uri_async), NULL); + ostree_fetcher_stream_mirrored_uri_async), NULL); return g_task_propagate_pointer (G_TASK (result), error); } @@ -1148,20 +1207,21 @@ fetch_uri_sync_on_complete (GObject *object, { FetchUriSyncData *data = user_data; - data->result_stream = ostree_fetcher_stream_uri_finish ((OstreeFetcher*)object, - result, data->error); + data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object, + result, data->error); data->done = TRUE; } gboolean -_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, - SoupURI *uri, - gboolean add_nul, - gboolean allow_noent, - GBytes **out_contents, - guint64 max_size, - GCancellable *cancellable, - GError **error) +_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, + GPtrArray *mirrorlist, + const char *filename, + gboolean add_nul, + gboolean allow_noent, + GBytes **out_contents, + guint64 max_size, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; const guint8 nulchar = 0; @@ -1182,10 +1242,8 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, data.done = FALSE; data.error = error; - ostree_fetcher_stream_uri_async (fetcher, uri, - max_size, - OSTREE_FETCHER_DEFAULT_PRIORITY, - cancellable, + ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size, + OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable, fetch_uri_sync_on_complete, &data); while (!data.done) g_main_context_iteration (mainctx, TRUE); @@ -1227,3 +1285,22 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, g_clear_object (&(data.result_stream)); return ret; } + +/* Helper for callers who just want to fetch single one-off URIs */ +gboolean +_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, + SoupURI *uri, + gboolean add_nul, + gboolean allow_noent, + GBytes **out_contents, + guint64 max_size, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); + g_ptr_array_add (mirrorlist, uri); /* no transfer */ + return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, + add_nul, allow_noent, + out_contents, max_size, + cancellable, error); +} diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 60a13755..8cceca51 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -70,20 +70,31 @@ void _ostree_fetcher_set_tls_database (OstreeFetcher *self, guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self); -void _ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self, - SoupURI *uri, - guint64 max_size, - int priority, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data); +void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); -char *_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self, - GAsyncResult *result, - GError **error); +char *_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self, + GAsyncResult *result, + GError **error); + +gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, + GPtrArray *mirrorlist, + const char *filename, + gboolean add_nul, + gboolean allow_noent, + GBytes **out_contents, + guint64 max_size, + GCancellable *cancellable, + GError **error); gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, - SoupURI *uri, + SoupURI *uri, gboolean add_nul, gboolean allow_noent, GBytes **out_contents, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 37a77cc3..2448d51e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -46,8 +46,8 @@ typedef struct { char *remote_name; OstreeRepoMode remote_mode; OstreeFetcher *fetcher; - SoupURI *base_uri; - SoupURI *base_content_uri; + GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ + GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ OstreeRepo *remote_repo_local; GMainContext *main_context; @@ -137,11 +137,6 @@ typedef struct { guint recursion_depth; } ScanObjectQueueData; -static SoupURI * -suburi_new (SoupURI *base, - const char *first, - ...) G_GNUC_NULL_TERMINATED; - static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, OstreeObjectType objtype, @@ -159,39 +154,6 @@ static gboolean scan_one_metadata_object_c (OtPullData *pull_data, GCancellable *cancellable, GError **error); -static SoupURI * -suburi_new (SoupURI *base, - const char *first, - ...) -{ - va_list args; - GPtrArray *arg_array; - const char *arg; - char *subpath; - SoupURI *ret; - - arg_array = g_ptr_array_new (); - g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base)); - g_ptr_array_add (arg_array, (char*)first); - - va_start (args, first); - - while ((arg = va_arg (args, const char *)) != NULL) - g_ptr_array_add (arg_array, (char*)arg); - g_ptr_array_add (arg_array, NULL); - - subpath = g_build_filenamev ((char**)arg_array->pdata); - g_ptr_array_unref (arg_array); - - ret = soup_uri_copy (base); - soup_uri_set_path (ret, subpath); - g_free (subpath); - - va_end (args); - - return ret; -} - static gboolean update_progress (gpointer user_data) { @@ -346,21 +308,23 @@ typedef struct { } OstreeFetchUriSyncData; static gboolean -fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher, - SoupURI *uri, - char **out_contents, - GCancellable *cancellable, - GError **error) +fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, + GPtrArray *mirrorlist, + const char *filename, + char **out_contents, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; g_autoptr(GBytes) bytes = NULL; g_autofree char *ret_contents = NULL; gsize len; - if (!_ostree_fetcher_request_uri_to_membuf (fetcher, uri, TRUE, - FALSE, &bytes, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) + if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, + filename, TRUE, FALSE, + &bytes, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; ret_contents = g_bytes_unref_to_data (bytes, &len); @@ -379,6 +343,20 @@ fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher, return ret; } +static gboolean +fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher, + SoupURI *uri, + char **out_contents, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); + g_ptr_array_add (mirrorlist, uri); /* no transfer */ + return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist, + NULL, out_contents, + cancellable, error); +} + static gboolean write_commitpartial_for (OtPullData *pull_data, const char *checksum, @@ -545,12 +523,14 @@ fetch_ref_contents (OtPullData *pull_data, { gboolean ret = FALSE; g_autofree char *ret_contents = NULL; - SoupURI *target_uri = NULL; + g_autofree char *filename = NULL; - target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL); + filename = g_build_filename ("refs", "heads", ref, NULL); - if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri, - &ret_contents, cancellable, error)) + if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, + pull_data->meta_mirrorlist, + filename, &ret_contents, + cancellable, error)) goto out; g_strchomp (ret_contents); @@ -561,8 +541,6 @@ fetch_ref_contents (OtPullData *pull_data, ret = TRUE; ot_transfer_out_value (out_contents, &ret_contents); out: - if (target_uri) - soup_uri_free (target_uri); return ret; } @@ -670,7 +648,7 @@ content_fetch_on_complete (GObject *object, OstreeObjectType objtype; gboolean free_fetch_data = TRUE; - temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error); + temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error); if (!temp_path) goto out; @@ -808,7 +786,7 @@ meta_fetch_on_complete (GObject *object, g_debug ("fetch of %s%s complete", checksum_obj, fetch_data->is_detached_meta ? " (detached)" : ""); - temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error); + temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error); if (!temp_path) { if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) @@ -953,7 +931,7 @@ static_deltapart_fetch_on_complete (GObject *object, g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum); - temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error); + temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error); if (!temp_path) goto out; @@ -1292,12 +1270,12 @@ enqueue_one_object_request (OtPullData *pull_data, gboolean is_detached_meta, gboolean object_is_stored) { - SoupURI *obj_uri = NULL; + g_autofree char *obj_subpath = NULL; gboolean is_meta; FetchObjectData *fetch_data; - g_autofree char *objpath = NULL; guint64 *expected_max_size_p; guint64 expected_max_size; + GPtrArray *mirrorlist = NULL; g_debug ("queuing fetch of %s.%s%s", checksum, ostree_object_type_to_string (objtype), @@ -1307,12 +1285,13 @@ enqueue_one_object_request (OtPullData *pull_data, { char buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode); - obj_uri = suburi_new (pull_data->base_uri, "objects", buf, NULL); + obj_subpath = g_build_filename ("objects", buf, NULL); + mirrorlist = pull_data->meta_mirrorlist; } else { - objpath = _ostree_get_relative_object_path (checksum, objtype, TRUE); - obj_uri = suburi_new (pull_data->base_content_uri, objpath, NULL); + obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE); + mirrorlist = pull_data->content_mirrorlist; } is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype); @@ -1340,13 +1319,12 @@ enqueue_one_object_request (OtPullData *pull_data, else expected_max_size = 0; - _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri, - expected_max_size, - is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY - : OSTREE_REPO_PULL_CONTENT_PRIORITY, - pull_data->cancellable, - is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data); - soup_uri_free (obj_uri); + _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist, + obj_subpath, expected_max_size, + is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY + : OSTREE_REPO_PULL_CONTENT_PRIORITY, + pull_data->cancellable, + is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data); } static gboolean @@ -1358,12 +1336,11 @@ load_remote_repo_config (OtPullData *pull_data, gboolean ret = FALSE; g_autofree char *contents = NULL; GKeyFile *ret_keyfile = NULL; - SoupURI *target_uri = NULL; - target_uri = suburi_new (pull_data->base_uri, "config", NULL); - - if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri, &contents, - cancellable, error)) + if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher, + pull_data->meta_mirrorlist, + "config", &contents, + cancellable, error)) goto out; ret_keyfile = g_key_file_new (); @@ -1375,7 +1352,6 @@ load_remote_repo_config (OtPullData *pull_data, ot_transfer_out_value (out_keyfile, &ret_keyfile); out: g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref); - g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free); return ret; } @@ -1394,17 +1370,15 @@ request_static_delta_superblock_sync (OtPullData *pull_data, g_autoptr(GBytes) delta_superblock_data = NULL; g_autoptr(GBytes) delta_meta_data = NULL; g_autoptr(GVariant) delta_superblock = NULL; - SoupURI *target_uri = NULL; - - target_uri = suburi_new (pull_data->base_content_uri, delta_name, NULL); - - if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, target_uri, - FALSE, TRUE, - &delta_superblock_data, - OSTREE_MAX_METADATA_SIZE, - pull_data->cancellable, error)) + + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, FALSE, TRUE, + &delta_superblock_data, + OSTREE_MAX_METADATA_SIZE, + pull_data->cancellable, error)) goto out; - + if (delta_superblock_data) { { @@ -1449,7 +1423,6 @@ request_static_delta_superblock_sync (OtPullData *pull_data, if (out_delta_superblock) *out_delta_superblock = g_steal_pointer (&ret_delta_superblock); out: - g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free); return ret; } @@ -1615,7 +1588,6 @@ process_one_static_delta (OtPullData *pull_data, const guchar *csum; g_autoptr(GVariant) header = NULL; gboolean have_all = FALSE; - SoupURI *target_uri = NULL; g_autofree char *deltapart_path = NULL; FetchStaticDeltaData *fetch_data; g_autoptr(GVariant) csum_v = NULL; @@ -1689,7 +1661,7 @@ process_one_static_delta (OtPullData *pull_data, NULL, &inline_delta_part, cancellable, error)) goto out; - + _ostree_static_delta_part_execute_async (pull_data->repo, fetch_data->objects, inline_delta_part, @@ -1701,14 +1673,14 @@ process_one_static_delta (OtPullData *pull_data, } else { - target_uri = suburi_new (pull_data->base_content_uri, deltapart_path, NULL); - _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size, - OSTREE_FETCHER_DEFAULT_PRIORITY, - pull_data->cancellable, - static_deltapart_fetch_on_complete, - fetch_data); + _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, + pull_data->content_mirrorlist, + deltapart_path, size, + OSTREE_FETCHER_DEFAULT_PRIORITY, + pull_data->cancellable, + static_deltapart_fetch_on_complete, + fetch_data); pull_data->n_outstanding_deltapart_fetches++; - soup_uri_free (target_uri); } } @@ -1947,7 +1919,7 @@ out: static gboolean _ostree_preload_metadata_file (OstreeRepo *self, OstreeFetcher *fetcher, - SoupURI *base_uri, + GPtrArray *mirrorlist, const char *filename, gboolean is_metalink, GBytes **out_bytes, @@ -1961,9 +1933,11 @@ _ostree_preload_metadata_file (OstreeRepo *self, glnx_unref_object OstreeMetalink *metalink = NULL; GError *local_error = NULL; + /* the metalink uri is buried in the mirrorlist as the first (and only) + * element */ metalink = _ostree_metalink_new (fetcher, filename, OSTREE_MAX_METADATA_SIZE, - base_uri); + mirrorlist->pdata[0]); _ostree_metalink_request_sync (metalink, NULL, out_bytes, cancellable, &local_error); @@ -1981,20 +1955,11 @@ _ostree_preload_metadata_file (OstreeRepo *self, } else { - SoupURI *uri; - const char *base_path; - g_autofree char *path = NULL; - - base_path = soup_uri_get_path (base_uri); - path = g_build_filename (base_path, filename, NULL); - uri = soup_uri_new_with_base (base_uri, path); - - ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri, - FALSE, TRUE, - out_bytes, - OSTREE_MAX_METADATA_SIZE, - cancellable, error); - soup_uri_free (uri); + ret = _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, + filename, FALSE, TRUE, + out_bytes, + OSTREE_MAX_METADATA_SIZE, + cancellable, error); if (!ret) goto out; @@ -2005,6 +1970,117 @@ out: return ret; } +static gboolean +fetch_mirrorlist (OstreeFetcher *fetcher, + const char *mirrorlist_url, + GPtrArray **out_mirrorlist, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + char **lines = NULL; + g_autofree char *contents = NULL; + SoupURI *mirrorlist = NULL; + g_autoptr(GPtrArray) ret_mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + + mirrorlist = soup_uri_new (mirrorlist_url); + if (mirrorlist == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse mirrorlist URL '%s'", mirrorlist_url); + goto out; + } + + if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents, + cancellable, error)) + { + g_prefix_error (error, "While fetching mirrorlist '%s': ", + mirrorlist_url); + goto out; + } + + /* go through each mirror in mirrorlist and do a quick sanity check that it + * works so that we don't waste the fetcher's time when it goes through them + * */ + lines = g_strsplit (contents, "\n", -1); + g_debug ("Scanning mirrorlist from '%s'", mirrorlist_url); + for (char **iter = lines; iter && *iter; iter++) + { + const char *mirror_uri_str = *iter; + SoupURI *mirror_uri = NULL; + + /* let's be nice and support empty lines and comments */ + if (*mirror_uri_str == '\0' || *mirror_uri_str == '#') + continue; + + mirror_uri = soup_uri_new (mirror_uri_str); + if (mirror_uri == NULL) + { + g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str); + continue; + } + else if ((strcmp (soup_uri_get_scheme (mirror_uri), "http") != 0) && + (strcmp (soup_uri_get_scheme (mirror_uri), "https") != 0)) + { + /* let's not support mirrorlists that contain non-http based URIs for + * now (e.g. local URIs) -- we need to think about if and how we want + * to support this since we set up things differently depending on + * whether we're pulling locally or not */ + g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str); + soup_uri_free (mirror_uri); + continue; + } + + /* We keep sanity checking until we hit a working mirror; there's no need + * to waste resources checking the remaining ones. At the same time, + * guaranteeing that the first mirror in the list works saves the fetcher + * time from always iterating through a few bad first mirrors. */ + if (ret_mirrorlist->len == 0) + { + GError *local_error = NULL; + g_autofree char *config_uri_str = g_build_filename (mirror_uri_str, + "config", NULL); + SoupURI *config_uri = soup_uri_new (config_uri_str); + + if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL, + cancellable, &local_error)) + g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri)); + else + { + g_debug ("Failed to fetch config from mirror '%s': %s", + mirror_uri_str, local_error->message); + g_clear_error (&local_error); + } + + soup_uri_free (config_uri); + } + else + { + g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri)); + } + + if (mirror_uri != NULL) + soup_uri_free (mirror_uri); + } + + if (ret_mirrorlist->len == 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No valid mirrors were found in mirrorlist '%s'", + mirrorlist_url); + goto out; + } + + *out_mirrorlist = g_steal_pointer (&ret_mirrorlist); + ret = TRUE; + +out: + if (mirrorlist != NULL) + soup_uri_free (mirrorlist); + return ret; +} + static gboolean repo_remote_fetch_summary (OstreeRepo *self, const char *name, @@ -2018,9 +2094,9 @@ repo_remote_fetch_summary (OstreeRepo *self, glnx_unref_object OstreeFetcher *fetcher = NULL; g_autoptr(GMainContext) mainctx = NULL; gboolean ret = FALSE; - SoupURI *base_uri = NULL; gboolean from_cache = FALSE; g_autofree char *url_override = NULL; + g_autoptr(GPtrArray) mirrorlist = NULL; if (options) (void) g_variant_lookup (options, "override-url", "&s", &url_override); @@ -2041,18 +2117,33 @@ repo_remote_fetch_summary (OstreeRepo *self, else if (!ostree_repo_remote_get_url (self, name, &url_string, error)) goto out; - base_uri = soup_uri_new (url_string); - if (base_uri == NULL) + if (metalink_url_string == NULL && + g_str_has_prefix (url_string, "mirrorlist=")) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid URL '%s'", url_string); - goto out; + if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="), + &mirrorlist, cancellable, error)) + goto out; + } + else + { + SoupURI *uri = soup_uri_new (url_string); + + if (uri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse url '%s'", url_string); + goto out; + } + + mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + g_ptr_array_add (mirrorlist, uri /* transfer ownership */ ); } } if (!_ostree_preload_metadata_file (self, fetcher, - base_uri, + mirrorlist, "summary.sig", metalink_url_string ? TRUE : FALSE, out_signatures, @@ -2077,7 +2168,7 @@ repo_remote_fetch_summary (OstreeRepo *self, { if (!_ostree_preload_metadata_file (self, fetcher, - base_uri, + mirrorlist, "summary", metalink_url_string ? TRUE : FALSE, out_summary, @@ -2112,8 +2203,6 @@ repo_remote_fetch_summary (OstreeRepo *self, out: if (mainctx) g_main_context_pop_thread_default (mainctx); - if (base_uri != NULL) - soup_uri_free (base_uri); return ret; } @@ -2181,6 +2270,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; const char *url_override = NULL; + g_autofree char *base_meta_url = NULL; + g_autofree char *base_content_url = NULL; if (options) { @@ -2305,13 +2396,28 @@ ostree_repo_pull_with_options (OstreeRepo *self, else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error)) goto out; - pull_data->base_uri = soup_uri_new (baseurl); - - if (!pull_data->base_uri) + if (g_str_has_prefix (baseurl, "mirrorlist=")) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse url '%s'", baseurl); - goto out; + if (!fetch_mirrorlist (pull_data->fetcher, + baseurl + strlen ("mirrorlist="), + &pull_data->meta_mirrorlist, + cancellable, error)) + goto out; + } + else + { + SoupURI *baseuri = soup_uri_new (baseurl); + + if (baseuri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse url '%s'", baseurl); + goto out; + } + + pull_data->meta_mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + g_ptr_array_add (pull_data->meta_mirrorlist, baseuri /* transfer */); } } else @@ -2338,10 +2444,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, error)) goto out; + /* XXX: would be interesting to implement metalink as another source of + * mirrors here since we use it as such anyway (rather than the "usual" + * use case of metalink, which is only for a single target filename) */ { + /* reuse target_uri and take ownership */ g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri)); - pull_data->base_uri = soup_uri_copy (target_uri); - soup_uri_set_path (pull_data->base_uri, repo_base); + soup_uri_set_path (target_uri, repo_base); + pull_data->meta_mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + g_ptr_array_add (pull_data->meta_mirrorlist, target_uri); } pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, @@ -2359,15 +2471,34 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; if (contenturl == NULL) - pull_data->base_content_uri = soup_uri_copy (pull_data->base_uri); + /* this is a bit hacky but greatly simplifies coding elsewhere; we take + * care in the out path to not double free if they're the same list */ + pull_data->content_mirrorlist = pull_data->meta_mirrorlist; else { - pull_data->base_content_uri = soup_uri_new (contenturl); - if (!pull_data->base_content_uri) + if (g_str_has_prefix (contenturl, "mirrorlist=")) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse contenturl '%s'", contenturl); - goto out; + if (!fetch_mirrorlist (pull_data->fetcher, + contenturl + strlen ("mirrorlist="), + &pull_data->content_mirrorlist, + cancellable, error)) + goto out; + } + else + { + SoupURI *contenturi = soup_uri_new (contenturl); + + if (contenturi == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse contenturl '%s'", contenturl); + goto out; + } + + pull_data->content_mirrorlist = + g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + g_ptr_array_add (pull_data->content_mirrorlist, + contenturi /* transfer */); } } } @@ -2377,9 +2508,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, &configured_branches, error)) goto out; - if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0) + /* NB: we don't support local mirrors in mirrorlists, so if this passes, it + * means that we're not using mirrorlists (see also fetch_mirrorlist()) */ + if (strcmp (soup_uri_get_scheme (pull_data->meta_mirrorlist->pdata[0]), "file") == 0) { - g_autoptr(GFile) remote_repo_path = g_file_new_for_path (soup_uri_get_path (pull_data->base_uri)); + g_autoptr(GFile) remote_repo_path = + g_file_new_for_path (soup_uri_get_path (pull_data->meta_mirrorlist->pdata[0])); pull_data->remote_repo_local = ostree_repo_new (remote_repo_path); if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error)) goto out; @@ -2418,7 +2552,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); { - SoupURI *uri = NULL; g_autoptr(GBytes) bytes_sig = NULL; g_autofree char *ret_contents = NULL; gsize i, n; @@ -2429,13 +2562,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!pull_data->summary_data_sig) { - uri = suburi_new (pull_data->base_uri, "summary.sig", NULL); - if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri, - FALSE, TRUE, &bytes_sig, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary.sig", FALSE, TRUE, + &bytes_sig, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - soup_uri_free (uri); } if (bytes_sig && @@ -2453,13 +2586,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!pull_data->summary && !bytes_summary) { - uri = suburi_new (pull_data->base_uri, "summary", NULL); - if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri, - FALSE, TRUE, &bytes_summary, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", FALSE, TRUE, + &bytes_summary, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - soup_uri_free (uri); } if (!bytes_summary && pull_data->gpg_verify_summary) @@ -2893,10 +3026,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_object (&pull_data->cancellable); g_clear_object (&pull_data->remote_repo_local); g_free (pull_data->remote_name); - if (pull_data->base_uri) - soup_uri_free (pull_data->base_uri); - if (pull_data->base_content_uri) - soup_uri_free (pull_data->base_content_uri); + if (pull_data->content_mirrorlist != pull_data->meta_mirrorlist) + g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref); + /* we clear this *after* clearing content_mirrorlist to avoid unref'ing twice */ + g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref); g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref); g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref); diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index 4e1d5595..76b0c75a 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -54,7 +54,7 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError g_autoptr(GVariantBuilder) optbuilder = NULL; gboolean ret = FALSE; - context = g_option_context_new ("NAME URL [BRANCH...] - Add a remote repository"); + context = g_option_context_new ("NAME [metalink=|mirrorlist=]URL [BRANCH...] - Add a remote repository"); if (!ostree_option_context_parse (context, option_entries, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))