pull: add mirrorlist support
This commit adds mirrorlist support to the fetcher. Users can now prepend url or/and contenturl by mirrorlist= to interpret the link as a mirrorlist. If an object is not found, the fetcher will automatically try the next mirror in the order given in the list (assuming the order returned by the server is significant). Closes: #469 Approved by: cgwalters
This commit is contained in:
parent
9546b93382
commit
157d878ce1
|
|
@ -74,7 +74,9 @@ typedef struct {
|
||||||
volatile int ref_count;
|
volatile int ref_count;
|
||||||
|
|
||||||
ThreadClosure *thread_closure;
|
ThreadClosure *thread_closure;
|
||||||
SoupURI *uri;
|
GPtrArray *mirrorlist; /* list of base URIs */
|
||||||
|
char *filename; /* relative name to fetch or NULL */
|
||||||
|
guint mirrorlist_idx;
|
||||||
|
|
||||||
OstreeFetcherState state;
|
OstreeFetcherState state;
|
||||||
|
|
||||||
|
|
@ -204,7 +206,8 @@ pending_uri_unref (OstreeFetcherPendingURI *pending)
|
||||||
|
|
||||||
g_clear_pointer (&pending->thread_closure, thread_closure_unref);
|
g_clear_pointer (&pending->thread_closure, thread_closure_unref);
|
||||||
|
|
||||||
soup_uri_free (pending->uri);
|
g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref);
|
||||||
|
g_free (pending->filename);
|
||||||
g_clear_object (&pending->request);
|
g_clear_object (&pending->request);
|
||||||
g_clear_object (&pending->request_body);
|
g_clear_object (&pending->request_body);
|
||||||
g_free (pending->out_tmpfile);
|
g_free (pending->out_tmpfile);
|
||||||
|
|
@ -353,6 +356,31 @@ session_thread_process_pending_queue (ThreadClosure *thread_closure)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
create_pending_soup_request (OstreeFetcherPendingURI *pending,
|
||||||
|
GError **error)
|
||||||
|
{
|
||||||
|
g_autofree char *uristr = NULL;
|
||||||
|
SoupURI *next_mirror = NULL;
|
||||||
|
SoupURI *uri = NULL;
|
||||||
|
|
||||||
|
g_assert (pending->mirrorlist);
|
||||||
|
g_assert (pending->mirrorlist_idx < pending->mirrorlist->len);
|
||||||
|
|
||||||
|
next_mirror = g_ptr_array_index (pending->mirrorlist,
|
||||||
|
pending->mirrorlist_idx);
|
||||||
|
uristr = g_build_filename (soup_uri_get_path (next_mirror),
|
||||||
|
pending->filename /* may be NULL */, NULL);
|
||||||
|
uri = soup_uri_copy (next_mirror);
|
||||||
|
soup_uri_set_path (uri, uristr);
|
||||||
|
|
||||||
|
g_clear_object (&pending->request);
|
||||||
|
|
||||||
|
pending->request = soup_session_request_uri (pending->thread_closure->session,
|
||||||
|
uri, error);
|
||||||
|
soup_uri_free (uri);
|
||||||
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
session_thread_request_uri (ThreadClosure *thread_closure,
|
session_thread_request_uri (ThreadClosure *thread_closure,
|
||||||
gpointer data)
|
gpointer data)
|
||||||
|
|
@ -365,10 +393,7 @@ session_thread_request_uri (ThreadClosure *thread_closure,
|
||||||
pending = g_task_get_task_data (task);
|
pending = g_task_get_task_data (task);
|
||||||
cancellable = g_task_get_cancellable (task);
|
cancellable = g_task_get_cancellable (task);
|
||||||
|
|
||||||
pending->request = soup_session_request_uri (thread_closure->session,
|
create_pending_soup_request (pending, &local_error);
|
||||||
pending->uri,
|
|
||||||
&local_error);
|
|
||||||
|
|
||||||
if (local_error != NULL)
|
if (local_error != NULL)
|
||||||
{
|
{
|
||||||
g_task_return_error (task, local_error);
|
g_task_return_error (task, local_error);
|
||||||
|
|
@ -384,7 +409,8 @@ session_thread_request_uri (ThreadClosure *thread_closure,
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
g_autofree char *uristring = soup_uri_to_string (pending->uri, FALSE);
|
g_autofree char *uristring
|
||||||
|
= soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
|
||||||
g_autofree char *tmpfile = NULL;
|
g_autofree char *tmpfile = NULL;
|
||||||
struct stat stbuf;
|
struct stat stbuf;
|
||||||
gboolean exists;
|
gboolean exists;
|
||||||
|
|
@ -463,6 +489,8 @@ ostree_fetcher_session_thread (gpointer data)
|
||||||
SOUP_SESSION_IDLE_TIMEOUT, 60,
|
SOUP_SESSION_IDLE_TIMEOUT, 60,
|
||||||
NULL);
|
NULL);
|
||||||
|
|
||||||
|
/* XXX: Now that we have mirrorlist support, we could make this even smarter
|
||||||
|
* by spreading requests across mirrors. */
|
||||||
g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
|
g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
|
||||||
if (max_conns < 8)
|
if (max_conns < 8)
|
||||||
{
|
{
|
||||||
|
|
@ -856,7 +884,8 @@ on_stream_read (GObject *object,
|
||||||
if (bytes_read > pending->max_size ||
|
if (bytes_read > pending->max_size ||
|
||||||
(bytes_read + pending->current_size) > pending->max_size)
|
(bytes_read + pending->current_size) > pending->max_size)
|
||||||
{
|
{
|
||||||
g_autofree char *uristr = soup_uri_to_string (pending->uri, FALSE);
|
g_autofree char *uristr =
|
||||||
|
soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
|
||||||
local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,
|
local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
"URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",
|
"URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",
|
||||||
uristr, pending->max_size);
|
uristr, pending->max_size);
|
||||||
|
|
@ -937,20 +966,43 @@ on_request_sent (GObject *object,
|
||||||
}
|
}
|
||||||
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
|
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
|
||||||
{
|
{
|
||||||
GIOErrorEnum code;
|
/* is there another mirror we can try? */
|
||||||
switch (msg->status_code)
|
if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len)
|
||||||
{
|
{
|
||||||
case 404:
|
pending->mirrorlist_idx++;
|
||||||
case 410:
|
create_pending_soup_request (pending, &local_error);
|
||||||
code = G_IO_ERROR_NOT_FOUND;
|
if (local_error != NULL)
|
||||||
break;
|
goto out;
|
||||||
default:
|
|
||||||
code = G_IO_ERROR_FAILED;
|
(void) g_input_stream_close (pending->request_body, NULL, NULL);
|
||||||
|
g_queue_insert_sorted (&pending->thread_closure->pending_queue,
|
||||||
|
g_object_ref (task), pending_task_compare,
|
||||||
|
NULL);
|
||||||
|
remove_pending_rerun_queue (pending);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
GIOErrorEnum code;
|
||||||
|
switch (msg->status_code)
|
||||||
|
{
|
||||||
|
case 404:
|
||||||
|
case 410:
|
||||||
|
code = G_IO_ERROR_NOT_FOUND;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
code = G_IO_ERROR_FAILED;
|
||||||
|
}
|
||||||
|
|
||||||
|
local_error = g_error_new (G_IO_ERROR, code,
|
||||||
|
"Server returned status %u: %s",
|
||||||
|
msg->status_code,
|
||||||
|
soup_status_get_phrase (msg->status_code));
|
||||||
|
|
||||||
|
if (pending->mirrorlist->len > 1)
|
||||||
|
g_prefix_error (&local_error,
|
||||||
|
"All %u mirrors failed. Last error was: ",
|
||||||
|
pending->mirrorlist->len);
|
||||||
}
|
}
|
||||||
local_error = g_error_new (G_IO_ERROR, code,
|
|
||||||
"Server returned status %u: %s",
|
|
||||||
msg->status_code,
|
|
||||||
soup_status_get_phrase (msg->status_code));
|
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1013,27 +1065,30 @@ on_request_sent (GObject *object,
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
ostree_fetcher_request_uri_internal (OstreeFetcher *self,
|
ostree_fetcher_mirrored_request_internal (OstreeFetcher *self,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
gboolean is_stream,
|
const char *filename,
|
||||||
guint64 max_size,
|
gboolean is_stream,
|
||||||
int priority,
|
guint64 max_size,
|
||||||
GCancellable *cancellable,
|
int priority,
|
||||||
GAsyncReadyCallback callback,
|
GCancellable *cancellable,
|
||||||
gpointer user_data,
|
GAsyncReadyCallback callback,
|
||||||
gpointer source_tag)
|
gpointer user_data,
|
||||||
|
gpointer source_tag)
|
||||||
{
|
{
|
||||||
g_autoptr(GTask) task = NULL;
|
g_autoptr(GTask) task = NULL;
|
||||||
OstreeFetcherPendingURI *pending;
|
OstreeFetcherPendingURI *pending;
|
||||||
|
|
||||||
g_return_if_fail (OSTREE_IS_FETCHER (self));
|
g_return_if_fail (OSTREE_IS_FETCHER (self));
|
||||||
g_return_if_fail (uri != NULL);
|
g_return_if_fail (mirrorlist != NULL);
|
||||||
|
g_return_if_fail (mirrorlist->len > 0);
|
||||||
|
|
||||||
/* SoupRequest is created in session thread. */
|
/* SoupRequest is created in session thread. */
|
||||||
pending = g_new0 (OstreeFetcherPendingURI, 1);
|
pending = g_new0 (OstreeFetcherPendingURI, 1);
|
||||||
pending->ref_count = 1;
|
pending->ref_count = 1;
|
||||||
pending->thread_closure = thread_closure_ref (self->thread_closure);
|
pending->thread_closure = thread_closure_ref (self->thread_closure);
|
||||||
pending->uri = soup_uri_copy (uri);
|
pending->mirrorlist = g_ptr_array_ref (mirrorlist);
|
||||||
|
pending->filename = g_strdup (filename);
|
||||||
pending->max_size = max_size;
|
pending->max_size = max_size;
|
||||||
pending->is_stream = is_stream;
|
pending->is_stream = is_stream;
|
||||||
|
|
||||||
|
|
@ -1051,53 +1106,57 @@ ostree_fetcher_request_uri_internal (OstreeFetcher *self,
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
_ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
|
_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
guint64 max_size,
|
const char *filename,
|
||||||
int priority,
|
guint64 max_size,
|
||||||
GCancellable *cancellable,
|
int priority,
|
||||||
GAsyncReadyCallback callback,
|
GCancellable *cancellable,
|
||||||
gpointer user_data)
|
GAsyncReadyCallback callback,
|
||||||
|
gpointer user_data)
|
||||||
{
|
{
|
||||||
ostree_fetcher_request_uri_internal (self, uri, FALSE, max_size, priority, cancellable,
|
ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE,
|
||||||
callback, user_data,
|
max_size, priority, cancellable,
|
||||||
_ostree_fetcher_request_uri_with_partial_async);
|
callback, user_data,
|
||||||
|
_ostree_fetcher_mirrored_request_with_partial_async);
|
||||||
}
|
}
|
||||||
|
|
||||||
char *
|
char *
|
||||||
_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
|
_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
|
||||||
GAsyncResult *result,
|
GAsyncResult *result,
|
||||||
GError **error)
|
GError **error)
|
||||||
{
|
{
|
||||||
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
|
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
|
||||||
g_return_val_if_fail (g_async_result_is_tagged (result,
|
g_return_val_if_fail (g_async_result_is_tagged (result,
|
||||||
_ostree_fetcher_request_uri_with_partial_async), NULL);
|
_ostree_fetcher_mirrored_request_with_partial_async), NULL);
|
||||||
|
|
||||||
return g_task_propagate_pointer (G_TASK (result), error);
|
return g_task_propagate_pointer (G_TASK (result), error);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
ostree_fetcher_stream_uri_async (OstreeFetcher *self,
|
ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher *self,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
guint64 max_size,
|
const char *filename,
|
||||||
int priority,
|
guint64 max_size,
|
||||||
GCancellable *cancellable,
|
int priority,
|
||||||
GAsyncReadyCallback callback,
|
GCancellable *cancellable,
|
||||||
gpointer user_data)
|
GAsyncReadyCallback callback,
|
||||||
|
gpointer user_data)
|
||||||
{
|
{
|
||||||
ostree_fetcher_request_uri_internal (self, uri, TRUE, max_size, priority, cancellable,
|
ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE,
|
||||||
callback, user_data,
|
max_size, priority, cancellable,
|
||||||
ostree_fetcher_stream_uri_async);
|
callback, user_data,
|
||||||
|
ostree_fetcher_stream_mirrored_uri_async);
|
||||||
}
|
}
|
||||||
|
|
||||||
static GInputStream *
|
static GInputStream *
|
||||||
ostree_fetcher_stream_uri_finish (OstreeFetcher *self,
|
ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher *self,
|
||||||
GAsyncResult *result,
|
GAsyncResult *result,
|
||||||
GError **error)
|
GError **error)
|
||||||
{
|
{
|
||||||
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
|
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
|
||||||
g_return_val_if_fail (g_async_result_is_tagged (result,
|
g_return_val_if_fail (g_async_result_is_tagged (result,
|
||||||
ostree_fetcher_stream_uri_async), NULL);
|
ostree_fetcher_stream_mirrored_uri_async), NULL);
|
||||||
|
|
||||||
return g_task_propagate_pointer (G_TASK (result), error);
|
return g_task_propagate_pointer (G_TASK (result), error);
|
||||||
}
|
}
|
||||||
|
|
@ -1148,20 +1207,21 @@ fetch_uri_sync_on_complete (GObject *object,
|
||||||
{
|
{
|
||||||
FetchUriSyncData *data = user_data;
|
FetchUriSyncData *data = user_data;
|
||||||
|
|
||||||
data->result_stream = ostree_fetcher_stream_uri_finish ((OstreeFetcher*)object,
|
data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object,
|
||||||
result, data->error);
|
result, data->error);
|
||||||
data->done = TRUE;
|
data->done = TRUE;
|
||||||
}
|
}
|
||||||
|
|
||||||
gboolean
|
gboolean
|
||||||
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
gboolean add_nul,
|
const char *filename,
|
||||||
gboolean allow_noent,
|
gboolean add_nul,
|
||||||
GBytes **out_contents,
|
gboolean allow_noent,
|
||||||
guint64 max_size,
|
GBytes **out_contents,
|
||||||
GCancellable *cancellable,
|
guint64 max_size,
|
||||||
GError **error)
|
GCancellable *cancellable,
|
||||||
|
GError **error)
|
||||||
{
|
{
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
const guint8 nulchar = 0;
|
const guint8 nulchar = 0;
|
||||||
|
|
@ -1182,10 +1242,8 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
||||||
data.done = FALSE;
|
data.done = FALSE;
|
||||||
data.error = error;
|
data.error = error;
|
||||||
|
|
||||||
ostree_fetcher_stream_uri_async (fetcher, uri,
|
ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size,
|
||||||
max_size,
|
OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
|
||||||
OSTREE_FETCHER_DEFAULT_PRIORITY,
|
|
||||||
cancellable,
|
|
||||||
fetch_uri_sync_on_complete, &data);
|
fetch_uri_sync_on_complete, &data);
|
||||||
while (!data.done)
|
while (!data.done)
|
||||||
g_main_context_iteration (mainctx, TRUE);
|
g_main_context_iteration (mainctx, TRUE);
|
||||||
|
|
@ -1227,3 +1285,22 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
||||||
g_clear_object (&(data.result_stream));
|
g_clear_object (&(data.result_stream));
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Helper for callers who just want to fetch single one-off URIs */
|
||||||
|
gboolean
|
||||||
|
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
||||||
|
SoupURI *uri,
|
||||||
|
gboolean add_nul,
|
||||||
|
gboolean allow_noent,
|
||||||
|
GBytes **out_contents,
|
||||||
|
guint64 max_size,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error)
|
||||||
|
{
|
||||||
|
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
|
||||||
|
g_ptr_array_add (mirrorlist, uri); /* no transfer */
|
||||||
|
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
|
||||||
|
add_nul, allow_noent,
|
||||||
|
out_contents, max_size,
|
||||||
|
cancellable, error);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -70,20 +70,31 @@ void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
|
||||||
|
|
||||||
guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self);
|
guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self);
|
||||||
|
|
||||||
void _ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
|
void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
guint64 max_size,
|
const char *filename,
|
||||||
int priority,
|
guint64 max_size,
|
||||||
GCancellable *cancellable,
|
int priority,
|
||||||
GAsyncReadyCallback callback,
|
GCancellable *cancellable,
|
||||||
gpointer user_data);
|
GAsyncReadyCallback callback,
|
||||||
|
gpointer user_data);
|
||||||
|
|
||||||
char *_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
|
char *_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
|
||||||
GAsyncResult *result,
|
GAsyncResult *result,
|
||||||
GError **error);
|
GError **error);
|
||||||
|
|
||||||
|
gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
|
||||||
|
GPtrArray *mirrorlist,
|
||||||
|
const char *filename,
|
||||||
|
gboolean add_nul,
|
||||||
|
gboolean allow_noent,
|
||||||
|
GBytes **out_contents,
|
||||||
|
guint64 max_size,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error);
|
||||||
|
|
||||||
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
|
||||||
SoupURI *uri,
|
SoupURI *uri,
|
||||||
gboolean add_nul,
|
gboolean add_nul,
|
||||||
gboolean allow_noent,
|
gboolean allow_noent,
|
||||||
GBytes **out_contents,
|
GBytes **out_contents,
|
||||||
|
|
|
||||||
|
|
@ -46,8 +46,8 @@ typedef struct {
|
||||||
char *remote_name;
|
char *remote_name;
|
||||||
OstreeRepoMode remote_mode;
|
OstreeRepoMode remote_mode;
|
||||||
OstreeFetcher *fetcher;
|
OstreeFetcher *fetcher;
|
||||||
SoupURI *base_uri;
|
GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */
|
||||||
SoupURI *base_content_uri;
|
GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */
|
||||||
OstreeRepo *remote_repo_local;
|
OstreeRepo *remote_repo_local;
|
||||||
|
|
||||||
GMainContext *main_context;
|
GMainContext *main_context;
|
||||||
|
|
@ -137,11 +137,6 @@ typedef struct {
|
||||||
guint recursion_depth;
|
guint recursion_depth;
|
||||||
} ScanObjectQueueData;
|
} ScanObjectQueueData;
|
||||||
|
|
||||||
static SoupURI *
|
|
||||||
suburi_new (SoupURI *base,
|
|
||||||
const char *first,
|
|
||||||
...) G_GNUC_NULL_TERMINATED;
|
|
||||||
|
|
||||||
static void queue_scan_one_metadata_object (OtPullData *pull_data,
|
static void queue_scan_one_metadata_object (OtPullData *pull_data,
|
||||||
const char *csum,
|
const char *csum,
|
||||||
OstreeObjectType objtype,
|
OstreeObjectType objtype,
|
||||||
|
|
@ -159,39 +154,6 @@ static gboolean scan_one_metadata_object_c (OtPullData *pull_data,
|
||||||
GCancellable *cancellable,
|
GCancellable *cancellable,
|
||||||
GError **error);
|
GError **error);
|
||||||
|
|
||||||
static SoupURI *
|
|
||||||
suburi_new (SoupURI *base,
|
|
||||||
const char *first,
|
|
||||||
...)
|
|
||||||
{
|
|
||||||
va_list args;
|
|
||||||
GPtrArray *arg_array;
|
|
||||||
const char *arg;
|
|
||||||
char *subpath;
|
|
||||||
SoupURI *ret;
|
|
||||||
|
|
||||||
arg_array = g_ptr_array_new ();
|
|
||||||
g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base));
|
|
||||||
g_ptr_array_add (arg_array, (char*)first);
|
|
||||||
|
|
||||||
va_start (args, first);
|
|
||||||
|
|
||||||
while ((arg = va_arg (args, const char *)) != NULL)
|
|
||||||
g_ptr_array_add (arg_array, (char*)arg);
|
|
||||||
g_ptr_array_add (arg_array, NULL);
|
|
||||||
|
|
||||||
subpath = g_build_filenamev ((char**)arg_array->pdata);
|
|
||||||
g_ptr_array_unref (arg_array);
|
|
||||||
|
|
||||||
ret = soup_uri_copy (base);
|
|
||||||
soup_uri_set_path (ret, subpath);
|
|
||||||
g_free (subpath);
|
|
||||||
|
|
||||||
va_end (args);
|
|
||||||
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
static gboolean
|
static gboolean
|
||||||
update_progress (gpointer user_data)
|
update_progress (gpointer user_data)
|
||||||
{
|
{
|
||||||
|
|
@ -346,21 +308,23 @@ typedef struct {
|
||||||
} OstreeFetchUriSyncData;
|
} OstreeFetchUriSyncData;
|
||||||
|
|
||||||
static gboolean
|
static gboolean
|
||||||
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
|
fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
|
||||||
SoupURI *uri,
|
GPtrArray *mirrorlist,
|
||||||
char **out_contents,
|
const char *filename,
|
||||||
GCancellable *cancellable,
|
char **out_contents,
|
||||||
GError **error)
|
GCancellable *cancellable,
|
||||||
|
GError **error)
|
||||||
{
|
{
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
g_autoptr(GBytes) bytes = NULL;
|
g_autoptr(GBytes) bytes = NULL;
|
||||||
g_autofree char *ret_contents = NULL;
|
g_autofree char *ret_contents = NULL;
|
||||||
gsize len;
|
gsize len;
|
||||||
|
|
||||||
if (!_ostree_fetcher_request_uri_to_membuf (fetcher, uri, TRUE,
|
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
|
||||||
FALSE, &bytes,
|
filename, TRUE, FALSE,
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
&bytes,
|
||||||
cancellable, error))
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
ret_contents = g_bytes_unref_to_data (bytes, &len);
|
ret_contents = g_bytes_unref_to_data (bytes, &len);
|
||||||
|
|
@ -379,6 +343,20 @@ fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
|
||||||
|
SoupURI *uri,
|
||||||
|
char **out_contents,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error)
|
||||||
|
{
|
||||||
|
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
|
||||||
|
g_ptr_array_add (mirrorlist, uri); /* no transfer */
|
||||||
|
return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
|
||||||
|
NULL, out_contents,
|
||||||
|
cancellable, error);
|
||||||
|
}
|
||||||
|
|
||||||
static gboolean
|
static gboolean
|
||||||
write_commitpartial_for (OtPullData *pull_data,
|
write_commitpartial_for (OtPullData *pull_data,
|
||||||
const char *checksum,
|
const char *checksum,
|
||||||
|
|
@ -545,12 +523,14 @@ fetch_ref_contents (OtPullData *pull_data,
|
||||||
{
|
{
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
g_autofree char *ret_contents = NULL;
|
g_autofree char *ret_contents = NULL;
|
||||||
SoupURI *target_uri = NULL;
|
g_autofree char *filename = NULL;
|
||||||
|
|
||||||
target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL);
|
filename = g_build_filename ("refs", "heads", ref, NULL);
|
||||||
|
|
||||||
if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri,
|
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
|
||||||
&ret_contents, cancellable, error))
|
pull_data->meta_mirrorlist,
|
||||||
|
filename, &ret_contents,
|
||||||
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
g_strchomp (ret_contents);
|
g_strchomp (ret_contents);
|
||||||
|
|
@ -561,8 +541,6 @@ fetch_ref_contents (OtPullData *pull_data,
|
||||||
ret = TRUE;
|
ret = TRUE;
|
||||||
ot_transfer_out_value (out_contents, &ret_contents);
|
ot_transfer_out_value (out_contents, &ret_contents);
|
||||||
out:
|
out:
|
||||||
if (target_uri)
|
|
||||||
soup_uri_free (target_uri);
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -670,7 +648,7 @@ content_fetch_on_complete (GObject *object,
|
||||||
OstreeObjectType objtype;
|
OstreeObjectType objtype;
|
||||||
gboolean free_fetch_data = TRUE;
|
gboolean free_fetch_data = TRUE;
|
||||||
|
|
||||||
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
|
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
|
||||||
if (!temp_path)
|
if (!temp_path)
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
|
|
@ -808,7 +786,7 @@ meta_fetch_on_complete (GObject *object,
|
||||||
g_debug ("fetch of %s%s complete", checksum_obj,
|
g_debug ("fetch of %s%s complete", checksum_obj,
|
||||||
fetch_data->is_detached_meta ? " (detached)" : "");
|
fetch_data->is_detached_meta ? " (detached)" : "");
|
||||||
|
|
||||||
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
|
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
|
||||||
if (!temp_path)
|
if (!temp_path)
|
||||||
{
|
{
|
||||||
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
|
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
|
||||||
|
|
@ -953,7 +931,7 @@ static_deltapart_fetch_on_complete (GObject *object,
|
||||||
|
|
||||||
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
|
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
|
||||||
|
|
||||||
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
|
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
|
||||||
if (!temp_path)
|
if (!temp_path)
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
|
|
@ -1292,12 +1270,12 @@ enqueue_one_object_request (OtPullData *pull_data,
|
||||||
gboolean is_detached_meta,
|
gboolean is_detached_meta,
|
||||||
gboolean object_is_stored)
|
gboolean object_is_stored)
|
||||||
{
|
{
|
||||||
SoupURI *obj_uri = NULL;
|
g_autofree char *obj_subpath = NULL;
|
||||||
gboolean is_meta;
|
gboolean is_meta;
|
||||||
FetchObjectData *fetch_data;
|
FetchObjectData *fetch_data;
|
||||||
g_autofree char *objpath = NULL;
|
|
||||||
guint64 *expected_max_size_p;
|
guint64 *expected_max_size_p;
|
||||||
guint64 expected_max_size;
|
guint64 expected_max_size;
|
||||||
|
GPtrArray *mirrorlist = NULL;
|
||||||
|
|
||||||
g_debug ("queuing fetch of %s.%s%s", checksum,
|
g_debug ("queuing fetch of %s.%s%s", checksum,
|
||||||
ostree_object_type_to_string (objtype),
|
ostree_object_type_to_string (objtype),
|
||||||
|
|
@ -1307,12 +1285,13 @@ enqueue_one_object_request (OtPullData *pull_data,
|
||||||
{
|
{
|
||||||
char buf[_OSTREE_LOOSE_PATH_MAX];
|
char buf[_OSTREE_LOOSE_PATH_MAX];
|
||||||
_ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
|
_ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
|
||||||
obj_uri = suburi_new (pull_data->base_uri, "objects", buf, NULL);
|
obj_subpath = g_build_filename ("objects", buf, NULL);
|
||||||
|
mirrorlist = pull_data->meta_mirrorlist;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
objpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
|
obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
|
||||||
obj_uri = suburi_new (pull_data->base_content_uri, objpath, NULL);
|
mirrorlist = pull_data->content_mirrorlist;
|
||||||
}
|
}
|
||||||
|
|
||||||
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
|
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
|
||||||
|
|
@ -1340,13 +1319,12 @@ enqueue_one_object_request (OtPullData *pull_data,
|
||||||
else
|
else
|
||||||
expected_max_size = 0;
|
expected_max_size = 0;
|
||||||
|
|
||||||
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri,
|
_ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist,
|
||||||
expected_max_size,
|
obj_subpath, expected_max_size,
|
||||||
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
|
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
|
||||||
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
|
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
|
||||||
pull_data->cancellable,
|
pull_data->cancellable,
|
||||||
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
|
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
|
||||||
soup_uri_free (obj_uri);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static gboolean
|
static gboolean
|
||||||
|
|
@ -1358,12 +1336,11 @@ load_remote_repo_config (OtPullData *pull_data,
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
g_autofree char *contents = NULL;
|
g_autofree char *contents = NULL;
|
||||||
GKeyFile *ret_keyfile = NULL;
|
GKeyFile *ret_keyfile = NULL;
|
||||||
SoupURI *target_uri = NULL;
|
|
||||||
|
|
||||||
target_uri = suburi_new (pull_data->base_uri, "config", NULL);
|
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
|
||||||
|
pull_data->meta_mirrorlist,
|
||||||
if (!fetch_uri_contents_utf8_sync (pull_data->fetcher, target_uri, &contents,
|
"config", &contents,
|
||||||
cancellable, error))
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
ret_keyfile = g_key_file_new ();
|
ret_keyfile = g_key_file_new ();
|
||||||
|
|
@ -1375,7 +1352,6 @@ load_remote_repo_config (OtPullData *pull_data,
|
||||||
ot_transfer_out_value (out_keyfile, &ret_keyfile);
|
ot_transfer_out_value (out_keyfile, &ret_keyfile);
|
||||||
out:
|
out:
|
||||||
g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref);
|
g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref);
|
||||||
g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1394,17 +1370,15 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
|
||||||
g_autoptr(GBytes) delta_superblock_data = NULL;
|
g_autoptr(GBytes) delta_superblock_data = NULL;
|
||||||
g_autoptr(GBytes) delta_meta_data = NULL;
|
g_autoptr(GBytes) delta_meta_data = NULL;
|
||||||
g_autoptr(GVariant) delta_superblock = NULL;
|
g_autoptr(GVariant) delta_superblock = NULL;
|
||||||
SoupURI *target_uri = NULL;
|
|
||||||
|
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
|
||||||
target_uri = suburi_new (pull_data->base_content_uri, delta_name, NULL);
|
pull_data->content_mirrorlist,
|
||||||
|
delta_name, FALSE, TRUE,
|
||||||
if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, target_uri,
|
&delta_superblock_data,
|
||||||
FALSE, TRUE,
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
&delta_superblock_data,
|
pull_data->cancellable, error))
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
|
||||||
pull_data->cancellable, error))
|
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
if (delta_superblock_data)
|
if (delta_superblock_data)
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
|
|
@ -1449,7 +1423,6 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
|
||||||
if (out_delta_superblock)
|
if (out_delta_superblock)
|
||||||
*out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
|
*out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
|
||||||
out:
|
out:
|
||||||
g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1615,7 +1588,6 @@ process_one_static_delta (OtPullData *pull_data,
|
||||||
const guchar *csum;
|
const guchar *csum;
|
||||||
g_autoptr(GVariant) header = NULL;
|
g_autoptr(GVariant) header = NULL;
|
||||||
gboolean have_all = FALSE;
|
gboolean have_all = FALSE;
|
||||||
SoupURI *target_uri = NULL;
|
|
||||||
g_autofree char *deltapart_path = NULL;
|
g_autofree char *deltapart_path = NULL;
|
||||||
FetchStaticDeltaData *fetch_data;
|
FetchStaticDeltaData *fetch_data;
|
||||||
g_autoptr(GVariant) csum_v = NULL;
|
g_autoptr(GVariant) csum_v = NULL;
|
||||||
|
|
@ -1689,7 +1661,7 @@ process_one_static_delta (OtPullData *pull_data,
|
||||||
NULL, &inline_delta_part,
|
NULL, &inline_delta_part,
|
||||||
cancellable, error))
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
_ostree_static_delta_part_execute_async (pull_data->repo,
|
_ostree_static_delta_part_execute_async (pull_data->repo,
|
||||||
fetch_data->objects,
|
fetch_data->objects,
|
||||||
inline_delta_part,
|
inline_delta_part,
|
||||||
|
|
@ -1701,14 +1673,14 @@ process_one_static_delta (OtPullData *pull_data,
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
target_uri = suburi_new (pull_data->base_content_uri, deltapart_path, NULL);
|
_ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher,
|
||||||
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
|
pull_data->content_mirrorlist,
|
||||||
OSTREE_FETCHER_DEFAULT_PRIORITY,
|
deltapart_path, size,
|
||||||
pull_data->cancellable,
|
OSTREE_FETCHER_DEFAULT_PRIORITY,
|
||||||
static_deltapart_fetch_on_complete,
|
pull_data->cancellable,
|
||||||
fetch_data);
|
static_deltapart_fetch_on_complete,
|
||||||
|
fetch_data);
|
||||||
pull_data->n_outstanding_deltapart_fetches++;
|
pull_data->n_outstanding_deltapart_fetches++;
|
||||||
soup_uri_free (target_uri);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1947,7 +1919,7 @@ out:
|
||||||
static gboolean
|
static gboolean
|
||||||
_ostree_preload_metadata_file (OstreeRepo *self,
|
_ostree_preload_metadata_file (OstreeRepo *self,
|
||||||
OstreeFetcher *fetcher,
|
OstreeFetcher *fetcher,
|
||||||
SoupURI *base_uri,
|
GPtrArray *mirrorlist,
|
||||||
const char *filename,
|
const char *filename,
|
||||||
gboolean is_metalink,
|
gboolean is_metalink,
|
||||||
GBytes **out_bytes,
|
GBytes **out_bytes,
|
||||||
|
|
@ -1961,9 +1933,11 @@ _ostree_preload_metadata_file (OstreeRepo *self,
|
||||||
glnx_unref_object OstreeMetalink *metalink = NULL;
|
glnx_unref_object OstreeMetalink *metalink = NULL;
|
||||||
GError *local_error = NULL;
|
GError *local_error = NULL;
|
||||||
|
|
||||||
|
/* the metalink uri is buried in the mirrorlist as the first (and only)
|
||||||
|
* element */
|
||||||
metalink = _ostree_metalink_new (fetcher, filename,
|
metalink = _ostree_metalink_new (fetcher, filename,
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
base_uri);
|
mirrorlist->pdata[0]);
|
||||||
|
|
||||||
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
|
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
|
||||||
cancellable, &local_error);
|
cancellable, &local_error);
|
||||||
|
|
@ -1981,20 +1955,11 @@ _ostree_preload_metadata_file (OstreeRepo *self,
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
SoupURI *uri;
|
ret = _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
|
||||||
const char *base_path;
|
filename, FALSE, TRUE,
|
||||||
g_autofree char *path = NULL;
|
out_bytes,
|
||||||
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
base_path = soup_uri_get_path (base_uri);
|
cancellable, error);
|
||||||
path = g_build_filename (base_path, filename, NULL);
|
|
||||||
uri = soup_uri_new_with_base (base_uri, path);
|
|
||||||
|
|
||||||
ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri,
|
|
||||||
FALSE, TRUE,
|
|
||||||
out_bytes,
|
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
|
||||||
cancellable, error);
|
|
||||||
soup_uri_free (uri);
|
|
||||||
|
|
||||||
if (!ret)
|
if (!ret)
|
||||||
goto out;
|
goto out;
|
||||||
|
|
@ -2005,6 +1970,117 @@ out:
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static gboolean
|
||||||
|
fetch_mirrorlist (OstreeFetcher *fetcher,
|
||||||
|
const char *mirrorlist_url,
|
||||||
|
GPtrArray **out_mirrorlist,
|
||||||
|
GCancellable *cancellable,
|
||||||
|
GError **error)
|
||||||
|
{
|
||||||
|
gboolean ret = FALSE;
|
||||||
|
char **lines = NULL;
|
||||||
|
g_autofree char *contents = NULL;
|
||||||
|
SoupURI *mirrorlist = NULL;
|
||||||
|
g_autoptr(GPtrArray) ret_mirrorlist =
|
||||||
|
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
|
||||||
|
|
||||||
|
mirrorlist = soup_uri_new (mirrorlist_url);
|
||||||
|
if (mirrorlist == NULL)
|
||||||
|
{
|
||||||
|
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"Failed to parse mirrorlist URL '%s'", mirrorlist_url);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
|
||||||
|
cancellable, error))
|
||||||
|
{
|
||||||
|
g_prefix_error (error, "While fetching mirrorlist '%s': ",
|
||||||
|
mirrorlist_url);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* go through each mirror in mirrorlist and do a quick sanity check that it
|
||||||
|
* works so that we don't waste the fetcher's time when it goes through them
|
||||||
|
* */
|
||||||
|
lines = g_strsplit (contents, "\n", -1);
|
||||||
|
g_debug ("Scanning mirrorlist from '%s'", mirrorlist_url);
|
||||||
|
for (char **iter = lines; iter && *iter; iter++)
|
||||||
|
{
|
||||||
|
const char *mirror_uri_str = *iter;
|
||||||
|
SoupURI *mirror_uri = NULL;
|
||||||
|
|
||||||
|
/* let's be nice and support empty lines and comments */
|
||||||
|
if (*mirror_uri_str == '\0' || *mirror_uri_str == '#')
|
||||||
|
continue;
|
||||||
|
|
||||||
|
mirror_uri = soup_uri_new (mirror_uri_str);
|
||||||
|
if (mirror_uri == NULL)
|
||||||
|
{
|
||||||
|
g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if ((strcmp (soup_uri_get_scheme (mirror_uri), "http") != 0) &&
|
||||||
|
(strcmp (soup_uri_get_scheme (mirror_uri), "https") != 0))
|
||||||
|
{
|
||||||
|
/* let's not support mirrorlists that contain non-http based URIs for
|
||||||
|
* now (e.g. local URIs) -- we need to think about if and how we want
|
||||||
|
* to support this since we set up things differently depending on
|
||||||
|
* whether we're pulling locally or not */
|
||||||
|
g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str);
|
||||||
|
soup_uri_free (mirror_uri);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* We keep sanity checking until we hit a working mirror; there's no need
|
||||||
|
* to waste resources checking the remaining ones. At the same time,
|
||||||
|
* guaranteeing that the first mirror in the list works saves the fetcher
|
||||||
|
* time from always iterating through a few bad first mirrors. */
|
||||||
|
if (ret_mirrorlist->len == 0)
|
||||||
|
{
|
||||||
|
GError *local_error = NULL;
|
||||||
|
g_autofree char *config_uri_str = g_build_filename (mirror_uri_str,
|
||||||
|
"config", NULL);
|
||||||
|
SoupURI *config_uri = soup_uri_new (config_uri_str);
|
||||||
|
|
||||||
|
if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
|
||||||
|
cancellable, &local_error))
|
||||||
|
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
|
||||||
|
else
|
||||||
|
{
|
||||||
|
g_debug ("Failed to fetch config from mirror '%s': %s",
|
||||||
|
mirror_uri_str, local_error->message);
|
||||||
|
g_clear_error (&local_error);
|
||||||
|
}
|
||||||
|
|
||||||
|
soup_uri_free (config_uri);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mirror_uri != NULL)
|
||||||
|
soup_uri_free (mirror_uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ret_mirrorlist->len == 0)
|
||||||
|
{
|
||||||
|
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"No valid mirrors were found in mirrorlist '%s'",
|
||||||
|
mirrorlist_url);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
*out_mirrorlist = g_steal_pointer (&ret_mirrorlist);
|
||||||
|
ret = TRUE;
|
||||||
|
|
||||||
|
out:
|
||||||
|
if (mirrorlist != NULL)
|
||||||
|
soup_uri_free (mirrorlist);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
static gboolean
|
static gboolean
|
||||||
repo_remote_fetch_summary (OstreeRepo *self,
|
repo_remote_fetch_summary (OstreeRepo *self,
|
||||||
const char *name,
|
const char *name,
|
||||||
|
|
@ -2018,9 +2094,9 @@ repo_remote_fetch_summary (OstreeRepo *self,
|
||||||
glnx_unref_object OstreeFetcher *fetcher = NULL;
|
glnx_unref_object OstreeFetcher *fetcher = NULL;
|
||||||
g_autoptr(GMainContext) mainctx = NULL;
|
g_autoptr(GMainContext) mainctx = NULL;
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
SoupURI *base_uri = NULL;
|
|
||||||
gboolean from_cache = FALSE;
|
gboolean from_cache = FALSE;
|
||||||
g_autofree char *url_override = NULL;
|
g_autofree char *url_override = NULL;
|
||||||
|
g_autoptr(GPtrArray) mirrorlist = NULL;
|
||||||
|
|
||||||
if (options)
|
if (options)
|
||||||
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
|
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
|
||||||
|
|
@ -2041,18 +2117,33 @@ repo_remote_fetch_summary (OstreeRepo *self,
|
||||||
else if (!ostree_repo_remote_get_url (self, name, &url_string, error))
|
else if (!ostree_repo_remote_get_url (self, name, &url_string, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
base_uri = soup_uri_new (url_string);
|
if (metalink_url_string == NULL &&
|
||||||
if (base_uri == NULL)
|
g_str_has_prefix (url_string, "mirrorlist="))
|
||||||
{
|
{
|
||||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="),
|
||||||
"Invalid URL '%s'", url_string);
|
&mirrorlist, cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
SoupURI *uri = soup_uri_new (url_string);
|
||||||
|
|
||||||
|
if (uri == NULL)
|
||||||
|
{
|
||||||
|
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"Failed to parse url '%s'", url_string);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
mirrorlist =
|
||||||
|
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
|
||||||
|
g_ptr_array_add (mirrorlist, uri /* transfer ownership */ );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!_ostree_preload_metadata_file (self,
|
if (!_ostree_preload_metadata_file (self,
|
||||||
fetcher,
|
fetcher,
|
||||||
base_uri,
|
mirrorlist,
|
||||||
"summary.sig",
|
"summary.sig",
|
||||||
metalink_url_string ? TRUE : FALSE,
|
metalink_url_string ? TRUE : FALSE,
|
||||||
out_signatures,
|
out_signatures,
|
||||||
|
|
@ -2077,7 +2168,7 @@ repo_remote_fetch_summary (OstreeRepo *self,
|
||||||
{
|
{
|
||||||
if (!_ostree_preload_metadata_file (self,
|
if (!_ostree_preload_metadata_file (self,
|
||||||
fetcher,
|
fetcher,
|
||||||
base_uri,
|
mirrorlist,
|
||||||
"summary",
|
"summary",
|
||||||
metalink_url_string ? TRUE : FALSE,
|
metalink_url_string ? TRUE : FALSE,
|
||||||
out_summary,
|
out_summary,
|
||||||
|
|
@ -2112,8 +2203,6 @@ repo_remote_fetch_summary (OstreeRepo *self,
|
||||||
out:
|
out:
|
||||||
if (mainctx)
|
if (mainctx)
|
||||||
g_main_context_pop_thread_default (mainctx);
|
g_main_context_pop_thread_default (mainctx);
|
||||||
if (base_uri != NULL)
|
|
||||||
soup_uri_free (base_uri);
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2181,6 +2270,8 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
gboolean opt_gpg_verify_set = FALSE;
|
gboolean opt_gpg_verify_set = FALSE;
|
||||||
gboolean opt_gpg_verify_summary_set = FALSE;
|
gboolean opt_gpg_verify_summary_set = FALSE;
|
||||||
const char *url_override = NULL;
|
const char *url_override = NULL;
|
||||||
|
g_autofree char *base_meta_url = NULL;
|
||||||
|
g_autofree char *base_content_url = NULL;
|
||||||
|
|
||||||
if (options)
|
if (options)
|
||||||
{
|
{
|
||||||
|
|
@ -2305,13 +2396,28 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error))
|
else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
pull_data->base_uri = soup_uri_new (baseurl);
|
if (g_str_has_prefix (baseurl, "mirrorlist="))
|
||||||
|
|
||||||
if (!pull_data->base_uri)
|
|
||||||
{
|
{
|
||||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
if (!fetch_mirrorlist (pull_data->fetcher,
|
||||||
"Failed to parse url '%s'", baseurl);
|
baseurl + strlen ("mirrorlist="),
|
||||||
goto out;
|
&pull_data->meta_mirrorlist,
|
||||||
|
cancellable, error))
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
SoupURI *baseuri = soup_uri_new (baseurl);
|
||||||
|
|
||||||
|
if (baseuri == NULL)
|
||||||
|
{
|
||||||
|
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"Failed to parse url '%s'", baseurl);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
pull_data->meta_mirrorlist =
|
||||||
|
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
|
||||||
|
g_ptr_array_add (pull_data->meta_mirrorlist, baseuri /* transfer */);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|
@ -2338,10 +2444,16 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
error))
|
error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
|
/* XXX: would be interesting to implement metalink as another source of
|
||||||
|
* mirrors here since we use it as such anyway (rather than the "usual"
|
||||||
|
* use case of metalink, which is only for a single target filename) */
|
||||||
{
|
{
|
||||||
|
/* reuse target_uri and take ownership */
|
||||||
g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
|
g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
|
||||||
pull_data->base_uri = soup_uri_copy (target_uri);
|
soup_uri_set_path (target_uri, repo_base);
|
||||||
soup_uri_set_path (pull_data->base_uri, repo_base);
|
pull_data->meta_mirrorlist =
|
||||||
|
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
|
||||||
|
g_ptr_array_add (pull_data->meta_mirrorlist, target_uri);
|
||||||
}
|
}
|
||||||
|
|
||||||
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
|
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
|
||||||
|
|
@ -2359,15 +2471,34 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
if (contenturl == NULL)
|
if (contenturl == NULL)
|
||||||
pull_data->base_content_uri = soup_uri_copy (pull_data->base_uri);
|
/* this is a bit hacky but greatly simplifies coding elsewhere; we take
|
||||||
|
* care in the out path to not double free if they're the same list */
|
||||||
|
pull_data->content_mirrorlist = pull_data->meta_mirrorlist;
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
pull_data->base_content_uri = soup_uri_new (contenturl);
|
if (g_str_has_prefix (contenturl, "mirrorlist="))
|
||||||
if (!pull_data->base_content_uri)
|
|
||||||
{
|
{
|
||||||
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
if (!fetch_mirrorlist (pull_data->fetcher,
|
||||||
"Failed to parse contenturl '%s'", contenturl);
|
contenturl + strlen ("mirrorlist="),
|
||||||
goto out;
|
&pull_data->content_mirrorlist,
|
||||||
|
cancellable, error))
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
SoupURI *contenturi = soup_uri_new (contenturl);
|
||||||
|
|
||||||
|
if (contenturi == NULL)
|
||||||
|
{
|
||||||
|
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
|
||||||
|
"Failed to parse contenturl '%s'", contenturl);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
pull_data->content_mirrorlist =
|
||||||
|
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
|
||||||
|
g_ptr_array_add (pull_data->content_mirrorlist,
|
||||||
|
contenturi /* transfer */);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2377,9 +2508,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
&configured_branches, error))
|
&configured_branches, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0)
|
/* NB: we don't support local mirrors in mirrorlists, so if this passes, it
|
||||||
|
* means that we're not using mirrorlists (see also fetch_mirrorlist()) */
|
||||||
|
if (strcmp (soup_uri_get_scheme (pull_data->meta_mirrorlist->pdata[0]), "file") == 0)
|
||||||
{
|
{
|
||||||
g_autoptr(GFile) remote_repo_path = g_file_new_for_path (soup_uri_get_path (pull_data->base_uri));
|
g_autoptr(GFile) remote_repo_path =
|
||||||
|
g_file_new_for_path (soup_uri_get_path (pull_data->meta_mirrorlist->pdata[0]));
|
||||||
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
|
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
|
||||||
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
|
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
@ -2418,7 +2552,6 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
|
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
|
||||||
|
|
||||||
{
|
{
|
||||||
SoupURI *uri = NULL;
|
|
||||||
g_autoptr(GBytes) bytes_sig = NULL;
|
g_autoptr(GBytes) bytes_sig = NULL;
|
||||||
g_autofree char *ret_contents = NULL;
|
g_autofree char *ret_contents = NULL;
|
||||||
gsize i, n;
|
gsize i, n;
|
||||||
|
|
@ -2429,13 +2562,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
|
|
||||||
if (!pull_data->summary_data_sig)
|
if (!pull_data->summary_data_sig)
|
||||||
{
|
{
|
||||||
uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
|
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
|
||||||
if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri,
|
pull_data->meta_mirrorlist,
|
||||||
FALSE, TRUE, &bytes_sig,
|
"summary.sig", FALSE, TRUE,
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
&bytes_sig,
|
||||||
cancellable, error))
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
soup_uri_free (uri);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bytes_sig &&
|
if (bytes_sig &&
|
||||||
|
|
@ -2453,13 +2586,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
|
|
||||||
if (!pull_data->summary && !bytes_summary)
|
if (!pull_data->summary && !bytes_summary)
|
||||||
{
|
{
|
||||||
uri = suburi_new (pull_data->base_uri, "summary", NULL);
|
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
|
||||||
if (!_ostree_fetcher_request_uri_to_membuf (pull_data->fetcher, uri,
|
pull_data->meta_mirrorlist,
|
||||||
FALSE, TRUE, &bytes_summary,
|
"summary", FALSE, TRUE,
|
||||||
OSTREE_MAX_METADATA_SIZE,
|
&bytes_summary,
|
||||||
cancellable, error))
|
OSTREE_MAX_METADATA_SIZE,
|
||||||
|
cancellable, error))
|
||||||
goto out;
|
goto out;
|
||||||
soup_uri_free (uri);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!bytes_summary && pull_data->gpg_verify_summary)
|
if (!bytes_summary && pull_data->gpg_verify_summary)
|
||||||
|
|
@ -2893,10 +3026,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
|
||||||
g_clear_object (&pull_data->cancellable);
|
g_clear_object (&pull_data->cancellable);
|
||||||
g_clear_object (&pull_data->remote_repo_local);
|
g_clear_object (&pull_data->remote_repo_local);
|
||||||
g_free (pull_data->remote_name);
|
g_free (pull_data->remote_name);
|
||||||
if (pull_data->base_uri)
|
if (pull_data->content_mirrorlist != pull_data->meta_mirrorlist)
|
||||||
soup_uri_free (pull_data->base_uri);
|
g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
|
||||||
if (pull_data->base_content_uri)
|
/* we clear this *after* clearing content_mirrorlist to avoid unref'ing twice */
|
||||||
soup_uri_free (pull_data->base_content_uri);
|
g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
|
||||||
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
|
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
|
||||||
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
|
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
|
||||||
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
|
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError
|
||||||
g_autoptr(GVariantBuilder) optbuilder = NULL;
|
g_autoptr(GVariantBuilder) optbuilder = NULL;
|
||||||
gboolean ret = FALSE;
|
gboolean ret = FALSE;
|
||||||
|
|
||||||
context = g_option_context_new ("NAME URL [BRANCH...] - Add a remote repository");
|
context = g_option_context_new ("NAME [metalink=|mirrorlist=]URL [BRANCH...] - Add a remote repository");
|
||||||
|
|
||||||
if (!ostree_option_context_parse (context, option_entries, &argc, &argv,
|
if (!ostree_option_context_parse (context, option_entries, &argc, &argv,
|
||||||
OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue