New upstream version 2016.10

This commit is contained in:
Simon McVittie 2016-09-09 16:56:45 +01:00
commit 9732c6c712
32 changed files with 845 additions and 362 deletions

View File

@ -40,7 +40,7 @@ man5_files = ostree.repo.5 ostree.repo-config.5
man1_MANS = $(addprefix man/,$(man1_files))
man5_MANS = $(addprefix man/,$(man5_files))
EXTRA_DIST += $(man1_MANS) $(man5_MANS)
EXTRA_DIST += $(man1_MANS) $(man5_MANS) $(man1_MANS:.1=.xml) $(man5_MANS:.5=.xml)
XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl

View File

@ -38,7 +38,7 @@ if BUILDOPT_USE_STATIC_COMPILER
ostree_boot_SCRIPTS = ostree-prepare-root
ostree-prepare-root : $(ostree_prepare_root_SOURCES)
$(STATIC_COMPILER) -o $@ -static $(ostree_prepare_root_SOURCES) $(AM_CPPFLAGS) $(AM_CFLAGS)
$(STATIC_COMPILER) -o $@ -static $(ostree_prepare_root_SOURCES) $(AM_CPPFLAGS) $(AM_CFLAGS) $(DEFAULT_INCLUDES)
else
ostree_boot_PROGRAMS += ostree-prepare-root

View File

@ -88,6 +88,8 @@ dist_test_scripts = \
tests/test-refs.sh \
tests/test-demo-buildsystem.sh \
tests/test-switchroot.sh \
tests/test-pull-contenturl.sh \
tests/test-pull-mirrorlist.sh \
$(NULL)
if BUILDOPT_FUSE

View File

@ -29,7 +29,11 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \
-DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \
-DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48
AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS)
AM_DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode
AM_DISTCHECK_CONFIGURE_FLAGS += \
--enable-gtk-doc \
--enable-man \
--disable-maintainer-mode \
$(NULL)
GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make

View File

@ -1,6 +1,6 @@
AC_PREREQ([2.63])
dnl If incrementing the version here, remember to update libostree.sym too
AC_INIT([ostree], [2016.9], [walters@verbum.org])
AC_INIT([ostree], [2016.10], [walters@verbum.org])
AC_CONFIG_HEADER([config.h])
AC_CONFIG_MACRO_DIR([buildutil])
AC_CONFIG_AUX_DIR([build-aux])

View File

@ -58,10 +58,11 @@ comparing timestamps. For Git, the logical choice is to not mess with
timestamps, because unnecessary rebuilding is better than a broken tree.
However, OSTree has to hardlink files to check them out, and commits are assumed
to be internally consistent with no build steps needed. For this reason, OSTree
acts as though all timestamps are set to time_t 1, so that comparisons will be
considered up-to-date. 1 is a better choice than 0 because some programs use 0
as a special value; for example, GNU Tar warns of an "implausibly old time
stamp" with 0.
acts as though all timestamps are set to time_t 0, so that comparisons will be
considered up-to-date. Note that for a few releases, OSTree used 1 to fix
warnings such as GNU Tar emitting "implausibly old time stamp" with 0; however,
until we have a mechanism to transition cleanly to 1, for compatibilty OSTree
is reverted to use zero again.
# Repository types and locations

View File

@ -354,6 +354,7 @@ global:
} LIBOSTREE_2016.7;
/* No new symbols in 2016.9 */
/* No new symbols in 2016.10 */
/* NOTE NOTE NOTE
* Versions above here are released. Only add symbols below this line.
@ -361,7 +362,7 @@ global:
*/
/* Remove comment when first new symbol is added
LIBOSTREE_2016.10
LIBOSTREE_2016.11
global:
someostree_symbol_deleteme;
} LIBOSTREE_2016.8;

View File

@ -165,6 +165,15 @@ typedef enum {
#define OSTREE_SUMMARY_SIG_GVARIANT_STRING "a{sv}"
#define OSTREE_SUMMARY_SIG_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING)
/**
* OSTREE_TIMESTAMP:
*
* The mtime used for stored files. This was originally 0, changed to 1 for
* a few releases, then was reverted due to regressions it introduced from
* users who had been using zero before.
*/
#define OSTREE_TIMESTAMP (0)
/**
* OstreeRepoMode:
* @OSTREE_REPO_MODE_BARE: Files are stored as themselves; checkouts are hardlinks; can only be written as root

View File

@ -46,7 +46,7 @@ typedef struct {
SoupSession *session; /* not referenced */
GMainContext *main_context;
GMainLoop *main_loop;
volatile gint running;
int tmpdir_dfd;
char *tmpdir_name;
@ -74,7 +74,9 @@ typedef struct {
volatile int ref_count;
ThreadClosure *thread_closure;
SoupURI *uri;
GPtrArray *mirrorlist; /* list of base URIs */
char *filename; /* relative name to fetch or NULL */
guint mirrorlist_idx;
OstreeFetcherState state;
@ -142,7 +144,6 @@ thread_closure_unref (ThreadClosure *thread_closure)
g_assert (thread_closure->session == NULL);
g_clear_pointer (&thread_closure->main_context, g_main_context_unref);
g_clear_pointer (&thread_closure->main_loop, g_main_loop_unref);
if (thread_closure->tmpdir_dfd != -1)
close (thread_closure->tmpdir_dfd);
@ -204,7 +205,8 @@ pending_uri_unref (OstreeFetcherPendingURI *pending)
g_clear_pointer (&pending->thread_closure, thread_closure_unref);
soup_uri_free (pending->uri);
g_clear_pointer (&pending->mirrorlist, g_ptr_array_unref);
g_free (pending->filename);
g_clear_object (&pending->request);
g_clear_object (&pending->request_body);
g_free (pending->out_tmpfile);
@ -353,6 +355,31 @@ session_thread_process_pending_queue (ThreadClosure *thread_closure)
}
}
static void
create_pending_soup_request (OstreeFetcherPendingURI *pending,
GError **error)
{
g_autofree char *uristr = NULL;
SoupURI *next_mirror = NULL;
SoupURI *uri = NULL;
g_assert (pending->mirrorlist);
g_assert (pending->mirrorlist_idx < pending->mirrorlist->len);
next_mirror = g_ptr_array_index (pending->mirrorlist,
pending->mirrorlist_idx);
uristr = g_build_filename (soup_uri_get_path (next_mirror),
pending->filename /* may be NULL */, NULL);
uri = soup_uri_copy (next_mirror);
soup_uri_set_path (uri, uristr);
g_clear_object (&pending->request);
pending->request = soup_session_request_uri (pending->thread_closure->session,
uri, error);
soup_uri_free (uri);
}
static void
session_thread_request_uri (ThreadClosure *thread_closure,
gpointer data)
@ -365,10 +392,7 @@ session_thread_request_uri (ThreadClosure *thread_closure,
pending = g_task_get_task_data (task);
cancellable = g_task_get_cancellable (task);
pending->request = soup_session_request_uri (thread_closure->session,
pending->uri,
&local_error);
create_pending_soup_request (pending, &local_error);
if (local_error != NULL)
{
g_task_return_error (task, local_error);
@ -384,7 +408,8 @@ session_thread_request_uri (ThreadClosure *thread_closure,
}
else
{
g_autofree char *uristring = soup_uri_to_string (pending->uri, FALSE);
g_autofree char *uristring
= soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
g_autofree char *tmpfile = NULL;
struct stat stbuf;
gboolean exists;
@ -463,6 +488,8 @@ ostree_fetcher_session_thread (gpointer data)
SOUP_SESSION_IDLE_TIMEOUT, 60,
NULL);
/* XXX: Now that we have mirrorlist support, we could make this even smarter
* by spreading requests across mirrors. */
g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
if (max_conns < 8)
{
@ -475,7 +502,12 @@ ostree_fetcher_session_thread (gpointer data)
}
closure->max_outstanding = 3 * max_conns;
g_main_loop_run (closure->main_loop);
/* This model ensures we don't hit a race using g_main_loop_quit();
* see also what pull_termination_condition() in ostree-repo-pull.c
* is doing.
*/
while (g_atomic_int_get (&closure->running))
g_main_context_iteration (closure->main_context, TRUE);
/* Since the ThreadClosure may be finalized from any thread we
* unreference all data related to the SoupSession ourself to ensure
@ -539,7 +571,8 @@ _ostree_fetcher_finalize (GObject *object)
OstreeFetcher *self = OSTREE_FETCHER (object);
/* Terminate the session thread. */
g_main_loop_quit (self->thread_closure->main_loop);
g_atomic_int_set (&self->thread_closure->running, 0);
g_main_context_wakeup (self->thread_closure->main_context);
if (self->session_thread)
{
/* We need to explicitly synchronize to clean up TLS */
@ -566,7 +599,7 @@ _ostree_fetcher_constructed (GObject *object)
self->thread_closure = g_slice_new0 (ThreadClosure);
self->thread_closure->ref_count = 1;
self->thread_closure->main_context = g_main_context_ref (main_context);
self->thread_closure->main_loop = g_main_loop_new (main_context, FALSE);
self->thread_closure->running = 1;
self->thread_closure->tmpdir_dfd = -1;
self->thread_closure->tmpdir_lock = empty_lockfile;
@ -856,7 +889,8 @@ on_stream_read (GObject *object,
if (bytes_read > pending->max_size ||
(bytes_read + pending->current_size) > pending->max_size)
{
g_autofree char *uristr = soup_uri_to_string (pending->uri, FALSE);
g_autofree char *uristr =
soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
local_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,
"URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",
uristr, pending->max_size);
@ -937,20 +971,43 @@ on_request_sent (GObject *object,
}
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
{
GIOErrorEnum code;
switch (msg->status_code)
/* is there another mirror we can try? */
if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len)
{
case 404:
case 410:
code = G_IO_ERROR_NOT_FOUND;
break;
default:
code = G_IO_ERROR_FAILED;
pending->mirrorlist_idx++;
create_pending_soup_request (pending, &local_error);
if (local_error != NULL)
goto out;
(void) g_input_stream_close (pending->request_body, NULL, NULL);
g_queue_insert_sorted (&pending->thread_closure->pending_queue,
g_object_ref (task), pending_task_compare,
NULL);
remove_pending_rerun_queue (pending);
}
else
{
GIOErrorEnum code;
switch (msg->status_code)
{
case 404:
case 410:
code = G_IO_ERROR_NOT_FOUND;
break;
default:
code = G_IO_ERROR_FAILED;
}
local_error = g_error_new (G_IO_ERROR, code,
"Server returned status %u: %s",
msg->status_code,
soup_status_get_phrase (msg->status_code));
if (pending->mirrorlist->len > 1)
g_prefix_error (&local_error,
"All %u mirrors failed. Last error was: ",
pending->mirrorlist->len);
}
local_error = g_error_new (G_IO_ERROR, code,
"Server returned status %u: %s",
msg->status_code,
soup_status_get_phrase (msg->status_code));
goto out;
}
}
@ -1013,27 +1070,30 @@ on_request_sent (GObject *object,
}
static void
ostree_fetcher_request_uri_internal (OstreeFetcher *self,
SoupURI *uri,
gboolean is_stream,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data,
gpointer source_tag)
ostree_fetcher_mirrored_request_internal (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
gboolean is_stream,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data,
gpointer source_tag)
{
g_autoptr(GTask) task = NULL;
OstreeFetcherPendingURI *pending;
g_return_if_fail (OSTREE_IS_FETCHER (self));
g_return_if_fail (uri != NULL);
g_return_if_fail (mirrorlist != NULL);
g_return_if_fail (mirrorlist->len > 0);
/* SoupRequest is created in session thread. */
pending = g_new0 (OstreeFetcherPendingURI, 1);
pending->ref_count = 1;
pending->thread_closure = thread_closure_ref (self->thread_closure);
pending->uri = soup_uri_copy (uri);
pending->mirrorlist = g_ptr_array_ref (mirrorlist);
pending->filename = g_strdup (filename);
pending->max_size = max_size;
pending->is_stream = is_stream;
@ -1051,53 +1111,57 @@ ostree_fetcher_request_uri_internal (OstreeFetcher *self,
}
void
_ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
SoupURI *uri,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
ostree_fetcher_request_uri_internal (self, uri, FALSE, max_size, priority, cancellable,
callback, user_data,
_ostree_fetcher_request_uri_with_partial_async);
ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE,
max_size, priority, cancellable,
callback, user_data,
_ostree_fetcher_mirrored_request_with_partial_async);
}
char *
_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error)
_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result,
_ostree_fetcher_request_uri_with_partial_async), NULL);
_ostree_fetcher_mirrored_request_with_partial_async), NULL);
return g_task_propagate_pointer (G_TASK (result), error);
}
static void
ostree_fetcher_stream_uri_async (OstreeFetcher *self,
SoupURI *uri,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
ostree_fetcher_request_uri_internal (self, uri, TRUE, max_size, priority, cancellable,
callback, user_data,
ostree_fetcher_stream_uri_async);
ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE,
max_size, priority, cancellable,
callback, user_data,
ostree_fetcher_stream_mirrored_uri_async);
}
static GInputStream *
ostree_fetcher_stream_uri_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error)
ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, self), NULL);
g_return_val_if_fail (g_async_result_is_tagged (result,
ostree_fetcher_stream_uri_async), NULL);
ostree_fetcher_stream_mirrored_uri_async), NULL);
return g_task_propagate_pointer (G_TASK (result), error);
}
@ -1148,20 +1212,21 @@ fetch_uri_sync_on_complete (GObject *object,
{
FetchUriSyncData *data = user_data;
data->result_stream = ostree_fetcher_stream_uri_finish ((OstreeFetcher*)object,
result, data->error);
data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object,
result, data->error);
data->done = TRUE;
}
gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
SoupURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
const guint8 nulchar = 0;
@ -1182,10 +1247,8 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
data.done = FALSE;
data.error = error;
ostree_fetcher_stream_uri_async (fetcher, uri,
max_size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
cancellable,
ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size,
OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
fetch_uri_sync_on_complete, &data);
while (!data.done)
g_main_context_iteration (mainctx, TRUE);
@ -1227,3 +1290,22 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
g_clear_object (&(data.result_stream));
return ret;
}
/* Helper for callers who just want to fetch single one-off URIs */
gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
SoupURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
add_nul, allow_noent,
out_contents, max_size,
cancellable, error);
}

View File

@ -70,20 +70,31 @@ void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self);
void _ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
SoupURI *uri,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data);
void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data);
char *_ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error);
char *_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
GAsyncResult *result,
GError **error);
gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
guint64 max_size,
GCancellable *cancellable,
GError **error);
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
SoupURI *uri,
SoupURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,

View File

@ -370,11 +370,15 @@ ostree_gpg_verify_result_get (OstreeGpgVerifyResult *result,
case OSTREE_GPG_SIGNATURE_ATTR_PUBKEY_ALGO_NAME:
v_string = gpgme_pubkey_algo_name (signature->pubkey_algo);
if (v_string == NULL)
v_string = "[unknown name]";
child = g_variant_new_string (v_string);
break;
case OSTREE_GPG_SIGNATURE_ATTR_HASH_ALGO_NAME:
v_string = gpgme_hash_algo_name (signature->hash_algo);
if (v_string == NULL)
v_string = "[unknown name]";
child = g_variant_new_string (v_string);
break;

View File

@ -593,7 +593,6 @@ gboolean
_ostree_metalink_request_sync (OstreeMetalink *self,
SoupURI **out_target_uri,
GBytes **out_data,
SoupURI **fetching_sync_uri,
GCancellable *cancellable,
GError **error)
{
@ -604,9 +603,6 @@ _ostree_metalink_request_sync (OstreeMetalink *self,
gsize len;
const guint8 *data;
if (fetching_sync_uri != NULL)
*fetching_sync_uri = _ostree_metalink_get_uri (self);
mainctx = g_main_context_new ();
g_main_context_push_thread_default (mainctx);

View File

@ -53,7 +53,6 @@ SoupURI *_ostree_metalink_get_uri (OstreeMetalink *self);
gboolean _ostree_metalink_request_sync (OstreeMetalink *self,
SoupURI **out_target_uri,
GBytes **out_data,
SoupURI **fetching_sync_uri,
GCancellable *cancellable,
GError **error);
G_END_DECLS

View File

@ -32,8 +32,6 @@ G_BEGIN_DECLS
#define _OSTREE_SUMMARY_CACHE_DIR "summaries"
#define _OSTREE_CACHE_DIR "cache"
#define OSTREE_TIMESTAMP (1)
typedef enum {
OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0)
} OstreeRepoTestErrorFlags;

View File

@ -46,7 +46,8 @@ typedef struct {
char *remote_name;
OstreeRepoMode remote_mode;
OstreeFetcher *fetcher;
SoupURI *base_uri;
GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */
GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */
OstreeRepo *remote_repo_local;
GMainContext *main_context;
@ -61,7 +62,6 @@ typedef struct {
OSTREE_PULL_PHASE_FETCHING_OBJECTS
} phase;
gint n_scanned_metadata;
SoupURI *fetching_sync_uri;
gboolean gpg_verify;
gboolean gpg_verify_summary;
@ -137,11 +137,6 @@ typedef struct {
guint recursion_depth;
} ScanObjectQueueData;
static SoupURI *
suburi_new (SoupURI *base,
const char *first,
...) G_GNUC_NULL_TERMINATED;
static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
OstreeObjectType objtype,
@ -159,39 +154,6 @@ static gboolean scan_one_metadata_object_c (OtPullData *pull_data,
GCancellable *cancellable,
GError **error);
static SoupURI *
suburi_new (SoupURI *base,
const char *first,
...)
{
va_list args;
GPtrArray *arg_array;
const char *arg;
char *subpath;
SoupURI *ret;
arg_array = g_ptr_array_new ();
g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base));
g_ptr_array_add (arg_array, (char*)first);
va_start (args, first);
while ((arg = va_arg (args, const char *)) != NULL)
g_ptr_array_add (arg_array, (char*)arg);
g_ptr_array_add (arg_array, NULL);
subpath = g_build_filenamev ((char**)arg_array->pdata);
g_ptr_array_unref (arg_array);
ret = soup_uri_copy (base);
soup_uri_set_path (ret, subpath);
g_free (subpath);
va_end (args);
return ret;
}
static gboolean
update_progress (gpointer user_data)
{
@ -245,14 +207,7 @@ update_progress (gpointer user_data)
ostree_async_progress_set_uint (pull_data->progress, "outstanding-metadata-fetches", pull_data->n_outstanding_metadata_fetches);
ostree_async_progress_set_uint (pull_data->progress, "metadata-fetched", pull_data->n_fetched_metadata);
if (pull_data->fetching_sync_uri)
{
g_autofree char *uri_string = soup_uri_to_string (pull_data->fetching_sync_uri, TRUE);
g_autofree char *status_string = g_strconcat ("Requesting ", uri_string, NULL);
ostree_async_progress_set_status (pull_data->progress, status_string);
}
else
ostree_async_progress_set_status (pull_data->progress, NULL);
ostree_async_progress_set_status (pull_data->progress, NULL);
if (pull_data->dry_run)
pull_data->dry_run_emitted_progress = TRUE;
@ -273,27 +228,19 @@ pull_termination_condition (OtPullData *pull_data)
gboolean current_scan_idle = g_queue_is_empty (&pull_data->scan_object_queue);
gboolean current_idle = current_fetch_idle && current_write_idle && current_scan_idle;
/* we only enter the main loop when we're fetching objects */
g_assert (pull_data->phase == OSTREE_PULL_PHASE_FETCHING_OBJECTS);
if (pull_data->caught_error)
return TRUE;
if (pull_data->dry_run)
return pull_data->dry_run_emitted_progress;
switch (pull_data->phase)
{
case OSTREE_PULL_PHASE_FETCHING_REFS:
if (!pull_data->fetching_sync_uri)
return TRUE;
break;
case OSTREE_PULL_PHASE_FETCHING_OBJECTS:
if (current_idle && !pull_data->fetching_sync_uri)
{
g_debug ("pull: idle, exiting mainloop");
return TRUE;
}
break;
}
return FALSE;
if (current_idle)
g_debug ("pull: idle, exiting mainloop");
return current_idle;
}
static void
@ -361,42 +308,23 @@ typedef struct {
} OstreeFetchUriSyncData;
static gboolean
fetch_uri_contents_membuf_sync (OtPullData *pull_data,
SoupURI *uri,
gboolean add_nul,
gboolean allow_noent,
GBytes **out_contents,
GCancellable *cancellable,
GError **error)
{
gboolean ret;
pull_data->fetching_sync_uri = uri;
ret = _ostree_fetcher_request_uri_to_membuf (pull_data->fetcher,
uri,
add_nul,
allow_noent,
out_contents,
OSTREE_MAX_METADATA_SIZE,
cancellable,
error);
pull_data->fetching_sync_uri = NULL;
return ret;
}
static gboolean
fetch_uri_contents_utf8_sync (OtPullData *pull_data,
SoupURI *uri,
char **out_contents,
GCancellable *cancellable,
GError **error)
fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
char **out_contents,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GBytes) bytes = NULL;
g_autofree char *ret_contents = NULL;
gsize len;
if (!fetch_uri_contents_membuf_sync (pull_data, uri, TRUE, FALSE,
&bytes, cancellable, error))
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, TRUE, FALSE,
&bytes,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
ret_contents = g_bytes_unref_to_data (bytes, &len);
@ -415,6 +343,20 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
return ret;
}
static gboolean
fetch_uri_contents_utf8_sync (OstreeFetcher *fetcher,
SoupURI *uri,
char **out_contents,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return fetch_mirrored_uri_contents_utf8_sync (fetcher, mirrorlist,
NULL, out_contents,
cancellable, error);
}
static gboolean
write_commitpartial_for (OtPullData *pull_data,
const char *checksum,
@ -581,11 +523,14 @@ fetch_ref_contents (OtPullData *pull_data,
{
gboolean ret = FALSE;
g_autofree char *ret_contents = NULL;
SoupURI *target_uri = NULL;
g_autofree char *filename = NULL;
target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL);
filename = g_build_filename ("refs", "heads", ref, NULL);
if (!fetch_uri_contents_utf8_sync (pull_data, target_uri, &ret_contents, cancellable, error))
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
filename, &ret_contents,
cancellable, error))
goto out;
g_strchomp (ret_contents);
@ -596,8 +541,6 @@ fetch_ref_contents (OtPullData *pull_data,
ret = TRUE;
ot_transfer_out_value (out_contents, &ret_contents);
out:
if (target_uri)
soup_uri_free (target_uri);
return ret;
}
@ -705,7 +648,7 @@ content_fetch_on_complete (GObject *object,
OstreeObjectType objtype;
gboolean free_fetch_data = TRUE;
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
goto out;
@ -843,7 +786,7 @@ meta_fetch_on_complete (GObject *object,
g_debug ("fetch of %s%s complete", checksum_obj,
fetch_data->is_detached_meta ? " (detached)" : "");
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@ -988,7 +931,7 @@ static_deltapart_fetch_on_complete (GObject *object,
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
temp_path = _ostree_fetcher_request_uri_with_partial_finish (fetcher, result, error);
temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
if (!temp_path)
goto out;
@ -1327,12 +1270,12 @@ enqueue_one_object_request (OtPullData *pull_data,
gboolean is_detached_meta,
gboolean object_is_stored)
{
SoupURI *obj_uri = NULL;
g_autofree char *obj_subpath = NULL;
gboolean is_meta;
FetchObjectData *fetch_data;
g_autofree char *objpath = NULL;
guint64 *expected_max_size_p;
guint64 expected_max_size;
GPtrArray *mirrorlist = NULL;
g_debug ("queuing fetch of %s.%s%s", checksum,
ostree_object_type_to_string (objtype),
@ -1342,12 +1285,13 @@ enqueue_one_object_request (OtPullData *pull_data,
{
char buf[_OSTREE_LOOSE_PATH_MAX];
_ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
obj_uri = suburi_new (pull_data->base_uri, "objects", buf, NULL);
obj_subpath = g_build_filename ("objects", buf, NULL);
mirrorlist = pull_data->meta_mirrorlist;
}
else
{
objpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
mirrorlist = pull_data->content_mirrorlist;
}
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
@ -1375,13 +1319,12 @@ enqueue_one_object_request (OtPullData *pull_data,
else
expected_max_size = 0;
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri,
expected_max_size,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
soup_uri_free (obj_uri);
_ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist,
obj_subpath, expected_max_size,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
}
static gboolean
@ -1393,12 +1336,11 @@ load_remote_repo_config (OtPullData *pull_data,
gboolean ret = FALSE;
g_autofree char *contents = NULL;
GKeyFile *ret_keyfile = NULL;
SoupURI *target_uri = NULL;
target_uri = suburi_new (pull_data->base_uri, "config", NULL);
if (!fetch_uri_contents_utf8_sync (pull_data, target_uri, &contents,
cancellable, error))
if (!fetch_mirrored_uri_contents_utf8_sync (pull_data->fetcher,
pull_data->meta_mirrorlist,
"config", &contents,
cancellable, error))
goto out;
ret_keyfile = g_key_file_new ();
@ -1410,7 +1352,6 @@ load_remote_repo_config (OtPullData *pull_data,
ot_transfer_out_value (out_keyfile, &ret_keyfile);
out:
g_clear_pointer (&ret_keyfile, (GDestroyNotify) g_key_file_unref);
g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
return ret;
}
@ -1429,13 +1370,13 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
g_autoptr(GBytes) delta_superblock_data = NULL;
g_autoptr(GBytes) delta_meta_data = NULL;
g_autoptr(GVariant) delta_superblock = NULL;
SoupURI *target_uri = NULL;
target_uri = suburi_new (pull_data->base_uri, delta_name, NULL);
if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, TRUE,
&delta_superblock_data,
pull_data->cancellable, error))
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->content_mirrorlist,
delta_name, FALSE, TRUE,
&delta_superblock_data,
OSTREE_MAX_METADATA_SIZE,
pull_data->cancellable, error))
goto out;
if (delta_superblock_data)
@ -1482,7 +1423,6 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
if (out_delta_superblock)
*out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
out:
g_clear_pointer (&target_uri, (GDestroyNotify) soup_uri_free);
return ret;
}
@ -1648,7 +1588,6 @@ process_one_static_delta (OtPullData *pull_data,
const guchar *csum;
g_autoptr(GVariant) header = NULL;
gboolean have_all = FALSE;
SoupURI *target_uri = NULL;
g_autofree char *deltapart_path = NULL;
FetchStaticDeltaData *fetch_data;
g_autoptr(GVariant) csum_v = NULL;
@ -1734,14 +1673,14 @@ process_one_static_delta (OtPullData *pull_data,
}
else
{
target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch_data);
_ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch_data);
pull_data->n_outstanding_deltapart_fetches++;
soup_uri_free (target_uri);
}
}
@ -1980,7 +1919,7 @@ out:
static gboolean
_ostree_preload_metadata_file (OstreeRepo *self,
OstreeFetcher *fetcher,
SoupURI *base_uri,
GPtrArray *mirrorlist,
const char *filename,
gboolean is_metalink,
GBytes **out_bytes,
@ -1994,11 +1933,13 @@ _ostree_preload_metadata_file (OstreeRepo *self,
glnx_unref_object OstreeMetalink *metalink = NULL;
GError *local_error = NULL;
/* the metalink uri is buried in the mirrorlist as the first (and only)
* element */
metalink = _ostree_metalink_new (fetcher, filename,
OSTREE_MAX_METADATA_SIZE,
base_uri);
mirrorlist->pdata[0]);
_ostree_metalink_request_sync (metalink, NULL, out_bytes, NULL,
_ostree_metalink_request_sync (metalink, NULL, out_bytes,
cancellable, &local_error);
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@ -2014,20 +1955,11 @@ _ostree_preload_metadata_file (OstreeRepo *self,
}
else
{
SoupURI *uri;
const char *base_path;
g_autofree char *path = NULL;
base_path = soup_uri_get_path (base_uri);
path = g_build_filename (base_path, filename, NULL);
uri = soup_uri_new_with_base (base_uri, path);
ret = _ostree_fetcher_request_uri_to_membuf (fetcher, uri,
FALSE, TRUE,
out_bytes,
OSTREE_MAX_METADATA_SIZE,
cancellable, error);
soup_uri_free (uri);
ret = _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, FALSE, TRUE,
out_bytes,
OSTREE_MAX_METADATA_SIZE,
cancellable, error);
if (!ret)
goto out;
@ -2038,6 +1970,117 @@ out:
return ret;
}
static gboolean
fetch_mirrorlist (OstreeFetcher *fetcher,
const char *mirrorlist_url,
GPtrArray **out_mirrorlist,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
char **lines = NULL;
g_autofree char *contents = NULL;
SoupURI *mirrorlist = NULL;
g_autoptr(GPtrArray) ret_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
mirrorlist = soup_uri_new (mirrorlist_url);
if (mirrorlist == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse mirrorlist URL '%s'", mirrorlist_url);
goto out;
}
if (!fetch_uri_contents_utf8_sync (fetcher, mirrorlist, &contents,
cancellable, error))
{
g_prefix_error (error, "While fetching mirrorlist '%s': ",
mirrorlist_url);
goto out;
}
/* go through each mirror in mirrorlist and do a quick sanity check that it
* works so that we don't waste the fetcher's time when it goes through them
* */
lines = g_strsplit (contents, "\n", -1);
g_debug ("Scanning mirrorlist from '%s'", mirrorlist_url);
for (char **iter = lines; iter && *iter; iter++)
{
const char *mirror_uri_str = *iter;
SoupURI *mirror_uri = NULL;
/* let's be nice and support empty lines and comments */
if (*mirror_uri_str == '\0' || *mirror_uri_str == '#')
continue;
mirror_uri = soup_uri_new (mirror_uri_str);
if (mirror_uri == NULL)
{
g_debug ("Can't parse mirrorlist line '%s'", mirror_uri_str);
continue;
}
else if ((strcmp (soup_uri_get_scheme (mirror_uri), "http") != 0) &&
(strcmp (soup_uri_get_scheme (mirror_uri), "https") != 0))
{
/* let's not support mirrorlists that contain non-http based URIs for
* now (e.g. local URIs) -- we need to think about if and how we want
* to support this since we set up things differently depending on
* whether we're pulling locally or not */
g_debug ("Ignoring non-http/s mirrorlist entry '%s'", mirror_uri_str);
soup_uri_free (mirror_uri);
continue;
}
/* We keep sanity checking until we hit a working mirror; there's no need
* to waste resources checking the remaining ones. At the same time,
* guaranteeing that the first mirror in the list works saves the fetcher
* time from always iterating through a few bad first mirrors. */
if (ret_mirrorlist->len == 0)
{
GError *local_error = NULL;
g_autofree char *config_uri_str = g_build_filename (mirror_uri_str,
"config", NULL);
SoupURI *config_uri = soup_uri_new (config_uri_str);
if (fetch_uri_contents_utf8_sync (fetcher, config_uri, NULL,
cancellable, &local_error))
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
else
{
g_debug ("Failed to fetch config from mirror '%s': %s",
mirror_uri_str, local_error->message);
g_clear_error (&local_error);
}
soup_uri_free (config_uri);
}
else
{
g_ptr_array_add (ret_mirrorlist, g_steal_pointer (&mirror_uri));
}
if (mirror_uri != NULL)
soup_uri_free (mirror_uri);
}
if (ret_mirrorlist->len == 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"No valid mirrors were found in mirrorlist '%s'",
mirrorlist_url);
goto out;
}
*out_mirrorlist = g_steal_pointer (&ret_mirrorlist);
ret = TRUE;
out:
if (mirrorlist != NULL)
soup_uri_free (mirrorlist);
return ret;
}
static gboolean
repo_remote_fetch_summary (OstreeRepo *self,
const char *name,
@ -2051,9 +2094,9 @@ repo_remote_fetch_summary (OstreeRepo *self,
glnx_unref_object OstreeFetcher *fetcher = NULL;
g_autoptr(GMainContext) mainctx = NULL;
gboolean ret = FALSE;
SoupURI *base_uri = NULL;
gboolean from_cache = FALSE;
g_autofree char *url_override = NULL;
g_autoptr(GPtrArray) mirrorlist = NULL;
if (options)
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
@ -2074,18 +2117,33 @@ repo_remote_fetch_summary (OstreeRepo *self,
else if (!ostree_repo_remote_get_url (self, name, &url_string, error))
goto out;
base_uri = soup_uri_new (url_string);
if (base_uri == NULL)
if (metalink_url_string == NULL &&
g_str_has_prefix (url_string, "mirrorlist="))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid URL '%s'", url_string);
goto out;
if (!fetch_mirrorlist (fetcher, url_string + strlen ("mirrorlist="),
&mirrorlist, cancellable, error))
goto out;
}
else
{
SoupURI *uri = soup_uri_new (url_string);
if (uri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", url_string);
goto out;
}
mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (mirrorlist, uri /* transfer ownership */ );
}
}
if (!_ostree_preload_metadata_file (self,
fetcher,
base_uri,
mirrorlist,
"summary.sig",
metalink_url_string ? TRUE : FALSE,
out_signatures,
@ -2110,7 +2168,7 @@ repo_remote_fetch_summary (OstreeRepo *self,
{
if (!_ostree_preload_metadata_file (self,
fetcher,
base_uri,
mirrorlist,
"summary",
metalink_url_string ? TRUE : FALSE,
out_summary,
@ -2145,8 +2203,6 @@ repo_remote_fetch_summary (OstreeRepo *self,
out:
if (mainctx)
g_main_context_pop_thread_default (mainctx);
if (base_uri != NULL)
soup_uri_free (base_uri);
return ret;
}
@ -2214,6 +2270,8 @@ ostree_repo_pull_with_options (OstreeRepo *self,
gboolean opt_gpg_verify_set = FALSE;
gboolean opt_gpg_verify_summary_set = FALSE;
const char *url_override = NULL;
g_autofree char *base_meta_url = NULL;
g_autofree char *base_content_url = NULL;
if (options)
{
@ -2338,13 +2396,28 @@ ostree_repo_pull_with_options (OstreeRepo *self,
else if (!ostree_repo_remote_get_url (self, remote_name_or_baseurl, &baseurl, error))
goto out;
pull_data->base_uri = soup_uri_new (baseurl);
if (!pull_data->base_uri)
if (g_str_has_prefix (baseurl, "mirrorlist="))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", baseurl);
goto out;
if (!fetch_mirrorlist (pull_data->fetcher,
baseurl + strlen ("mirrorlist="),
&pull_data->meta_mirrorlist,
cancellable, error))
goto out;
}
else
{
SoupURI *baseuri = soup_uri_new (baseurl);
if (baseuri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", baseurl);
goto out;
}
pull_data->meta_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, baseuri /* transfer */);
}
}
else
@ -2367,29 +2440,81 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (! _ostree_metalink_request_sync (metalink,
&target_uri,
&summary_bytes,
&pull_data->fetching_sync_uri,
cancellable,
error))
goto out;
/* XXX: would be interesting to implement metalink as another source of
* mirrors here since we use it as such anyway (rather than the "usual"
* use case of metalink, which is only for a single target filename) */
{
/* reuse target_uri and take ownership */
g_autofree char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
pull_data->base_uri = soup_uri_copy (target_uri);
soup_uri_set_path (pull_data->base_uri, repo_base);
soup_uri_set_path (target_uri, repo_base);
pull_data->meta_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (pull_data->meta_mirrorlist, target_uri);
}
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT,
summary_bytes, FALSE);
}
{
g_autofree char *contenturl = NULL;
if (metalink_url_str == NULL && url_override != NULL)
contenturl = g_strdup (url_override);
else if (!ostree_repo_get_remote_option (self, remote_name_or_baseurl,
"contenturl", NULL,
&contenturl, error))
goto out;
if (contenturl == NULL)
{
pull_data->content_mirrorlist =
g_ptr_array_ref (pull_data->meta_mirrorlist);
}
else
{
if (g_str_has_prefix (contenturl, "mirrorlist="))
{
if (!fetch_mirrorlist (pull_data->fetcher,
contenturl + strlen ("mirrorlist="),
&pull_data->content_mirrorlist,
cancellable, error))
goto out;
}
else
{
SoupURI *contenturi = soup_uri_new (contenturl);
if (contenturi == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse contenturl '%s'", contenturl);
goto out;
}
pull_data->content_mirrorlist =
g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
g_ptr_array_add (pull_data->content_mirrorlist,
contenturi /* transfer */);
}
}
}
if (!ostree_repo_get_remote_list_option (self,
remote_name_or_baseurl, "branches",
&configured_branches, error))
goto out;
if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0)
/* NB: we don't support local mirrors in mirrorlists, so if this passes, it
* means that we're not using mirrorlists (see also fetch_mirrorlist()) */
if (strcmp (soup_uri_get_scheme (pull_data->meta_mirrorlist->pdata[0]), "file") == 0)
{
g_autoptr(GFile) remote_repo_path = g_file_new_for_path (soup_uri_get_path (pull_data->base_uri));
g_autoptr(GFile) remote_repo_path =
g_file_new_for_path (soup_uri_get_path (pull_data->meta_mirrorlist->pdata[0]));
pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
goto out;
@ -2428,7 +2553,6 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
{
SoupURI *uri = NULL;
g_autoptr(GBytes) bytes_sig = NULL;
g_autofree char *ret_contents = NULL;
gsize i, n;
@ -2439,11 +2563,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!pull_data->summary_data_sig)
{
uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
&bytes_sig, cancellable, error))
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", FALSE, TRUE,
&bytes_sig,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
soup_uri_free (uri);
}
if (bytes_sig &&
@ -2461,11 +2587,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!pull_data->summary && !bytes_summary)
{
uri = suburi_new (pull_data->base_uri, "summary", NULL);
if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
&bytes_summary, cancellable, error))
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", FALSE, TRUE,
&bytes_summary,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
soup_uri_free (uri);
}
if (!bytes_summary && pull_data->gpg_verify_summary)
@ -2899,8 +3027,8 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_clear_object (&pull_data->cancellable);
g_clear_object (&pull_data->remote_repo_local);
g_free (pull_data->remote_name);
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);

View File

@ -429,8 +429,8 @@ content_rollsums_free (ContentRollsum *rollsum)
{
g_free (rollsum->from_checksum);
_ostree_rollsum_matches_free (rollsum->matches);
g_bytes_unref (rollsum->tmp_from);
g_bytes_unref (rollsum->tmp_to);
g_clear_pointer (&rollsum->tmp_from, g_bytes_unref);
g_clear_pointer (&rollsum->tmp_to, g_bytes_unref);
g_free (rollsum);
}
@ -438,8 +438,8 @@ static void
content_bsdiffs_free (ContentBsdiff *bsdiff)
{
g_free (bsdiff->from_checksum);
g_bytes_unref (bsdiff->tmp_from);
g_bytes_unref (bsdiff->tmp_to);
g_clear_pointer (&bsdiff->tmp_from, g_bytes_unref);
g_clear_pointer (&bsdiff->tmp_to, g_bytes_unref);
g_free (bsdiff);
}
@ -483,6 +483,8 @@ get_unpacked_unlinked_content (OstreeRepo *repo,
goto out;
{ GMappedFile *mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
if (!mfile)
goto out;
ret_content = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
}
@ -756,6 +758,9 @@ process_one_rollsum (OstreeRepo *repo,
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
}
g_clear_pointer (&rollsum->tmp_from, g_bytes_unref);
g_clear_pointer (&rollsum->tmp_to, g_bytes_unref);
ret = TRUE;
out:
return ret;
@ -852,6 +857,9 @@ process_one_bsdiff (OstreeRepo *repo,
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE);
g_clear_pointer (&bsdiff_content->tmp_from, g_bytes_unref);
g_clear_pointer (&bsdiff_content->tmp_to, g_bytes_unref);
ret = TRUE;
out:
return ret;

View File

@ -2544,6 +2544,7 @@ load_metadata_internal (OstreeRepo *self,
{
gboolean ret = FALSE;
char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
struct stat stbuf;
glnx_fd_close int fd = -1;
g_autoptr(GInputStream) ret_stream = NULL;
g_autoptr(GVariant) ret_variant = NULL;
@ -2565,23 +2566,39 @@ load_metadata_internal (OstreeRepo *self,
if (fd != -1)
{
if (fstat (fd, &stbuf) < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
if (out_variant)
{
GMappedFile *mfile;
/* http://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access */
if (stbuf.st_size > 16*1024)
{
GMappedFile *mfile;
mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
if (!mfile)
goto out;
ret_variant = g_variant_new_from_data (ostree_metadata_variant_type (objtype),
g_mapped_file_get_contents (mfile),
g_mapped_file_get_length (mfile),
TRUE,
(GDestroyNotify) g_mapped_file_unref,
mfile);
g_variant_ref_sink (ret_variant);
if (out_size)
*out_size = g_variant_get_size (ret_variant);
mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
if (!mfile)
goto out;
ret_variant = g_variant_new_from_data (ostree_metadata_variant_type (objtype),
g_mapped_file_get_contents (mfile),
g_mapped_file_get_length (mfile),
TRUE,
(GDestroyNotify) g_mapped_file_unref,
mfile);
g_variant_ref_sink (ret_variant);
}
else
{
GBytes *data = glnx_fd_readall_bytes (fd, cancellable, error);
if (!data)
goto out;
ret_variant = g_variant_new_from_bytes (ostree_metadata_variant_type (objtype),
data, TRUE);
g_variant_ref_sink (ret_variant);
}
}
else if (out_stream)
{
@ -2589,15 +2606,10 @@ load_metadata_internal (OstreeRepo *self,
if (!ret_stream)
goto out;
fd = -1; /* Transfer ownership */
if (out_size)
{
struct stat stbuf;
if (!glnx_stream_fstat ((GFileDescriptorBased*)ret_stream, &stbuf, error))
goto out;
*out_size = stbuf.st_size;
}
}
if (out_size)
*out_size = stbuf.st_size;
}
else if (self->parent_repo)
{

View File

@ -1712,6 +1712,18 @@ ostree_sysroot_write_deployments (OstreeSysroot *self,
GPtrArray *new_deployments,
GCancellable *cancellable,
GError **error)
{
return _ostree_sysroot_write_deployments_internal (self, new_deployments,
OSTREE_SYSROOT_CLEANUP_ALL,
cancellable, error);
}
gboolean
_ostree_sysroot_write_deployments_internal (OstreeSysroot *self,
GPtrArray *new_deployments,
OstreeSysrootCleanupFlags cleanup_flags,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
@ -1937,7 +1949,8 @@ ostree_sysroot_write_deployments (OstreeSysroot *self,
/* And finally, cleanup of any leftover data.
*/
if (!ostree_sysroot_cleanup (self, cancellable, error))
if (!_ostree_sysroot_piecemeal_cleanup (self, cleanup_flags,
cancellable, error))
{
g_prefix_error (error, "Performing final cleanup: ");
goto out;

View File

@ -121,4 +121,10 @@ gboolean _ostree_sysroot_piecemeal_cleanup (OstreeSysroot *sysroot,
GCancellable *cancellable,
GError **error);
gboolean _ostree_sysroot_write_deployments_internal (OstreeSysroot *self,
GPtrArray *new_deployments,
OstreeSysrootCleanupFlags cleanup_flags,
GCancellable *cancellable,
GError **error);
G_END_DECLS

View File

@ -1591,15 +1591,11 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot,
added_new = TRUE;
}
if (!ostree_sysroot_write_deployments (sysroot, new_deployments, cancellable, error))
if (!_ostree_sysroot_write_deployments_internal (sysroot, new_deployments,
postclean ? OSTREE_SYSROOT_CLEANUP_ALL : 0,
cancellable, error))
goto out;
if (postclean)
{
if (!ostree_sysroot_cleanup (sysroot, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;

View File

@ -77,9 +77,15 @@ httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...)
if (!httpd->log)
return;
str = g_string_new (NULL);
{
g_autoptr(GDateTime) now = g_date_time_new_now_local ();
g_autofree char *timestamp = g_date_time_format (now, "%F %T");
str = g_string_new (timestamp);
g_string_append_printf (str, ".%06d - ", g_date_time_get_microsecond (now));
}
va_start (args, format);
g_string_vprintf (str, format, args);
g_string_append_vprintf (str, format, args);
va_end (args);
g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL);

View File

@ -30,12 +30,14 @@ static char **opt_set;
static gboolean opt_no_gpg_verify;
static gboolean opt_if_not_exists;
static char *opt_gpg_import;
static char *opt_contenturl;
static GOptionEntry option_entries[] = {
{ "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" },
{ "no-gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_no_gpg_verify, "Disable GPG verification", NULL },
{ "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL },
{ "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" },
{ "contenturl", 0, 0, G_OPTION_ARG_STRING, &opt_contenturl, "Use URL when fetching content", "URL" },
{ NULL }
};
@ -52,7 +54,7 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError
g_autoptr(GVariantBuilder) optbuilder = NULL;
gboolean ret = FALSE;
context = g_option_context_new ("NAME URL [BRANCH...] - Add a remote repository");
context = g_option_context_new ("NAME [metalink=|mirrorlist=]URL [BRANCH...] - Add a remote repository");
if (!ostree_option_context_parse (context, option_entries, &argc, &argv,
OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
@ -83,6 +85,14 @@ ot_remote_builtin_add (int argc, char **argv, GCancellable *cancellable, GError
g_variant_new_variant (g_variant_new_strv ((const char*const*)branchesp->pdata, -1)));
}
/* We could just make users use --set instead for this since it's a string,
* but e.g. when mirrorlist support is added, it'll be kinda awkward to type:
* --set=contenturl=mirrorlist=... */
if (opt_contenturl != NULL)
g_variant_builder_add (optbuilder, "{s@v}",
"contenturl", g_variant_new_variant (g_variant_new_string (opt_contenturl)));
for (iter = opt_set; iter && *iter; iter++)
{
const char *keyvalue = *iter;

View File

@ -80,26 +80,10 @@ parse_ostree_cmdline (void)
char *cmdline = NULL;
const char *iter;
char *ret = NULL;
int tmp_errno;
cmdline = read_proc_cmdline ();
if (!cmdline)
{
// Mount proc
if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
err (EXIT_FAILURE, "failed to mount proc on /proc");
cmdline = read_proc_cmdline ();
tmp_errno = errno;
/* Leave the filesystem in the state that we found it: */
if (umount ("/proc"))
err (EXIT_FAILURE, "failed to umount proc from /proc");
errno = tmp_errno;
if (!cmdline)
err (EXIT_FAILURE, "failed to read /proc/cmdline");
}
err (EXIT_FAILURE, "failed to read /proc/cmdline");
iter = cmdline;
while (iter != NULL)
@ -174,19 +158,40 @@ pivot_root(const char * new_root, const char * put_old)
int
main(int argc, char *argv[])
{
const char *root_mountpoint = NULL;
const char *root_mountpoint = NULL, *root_arg = NULL;
char *deploy_path = NULL;
char srcpath[PATH_MAX];
struct stat stbuf;
int we_mounted_proc = 0;
if (argc < 2)
root_mountpoint = "/";
root_arg = "/";
else
root_mountpoint = argv[1];
root_arg = argv[1];
root_mountpoint = realpath (root_mountpoint, NULL);
if (stat ("/proc/cmdline", &stbuf) < 0)
{
if (errno != ENOENT)
err (EXIT_FAILURE, "stat(\"/proc/cmdline\") failed");
/* We need /proc mounted for /proc/cmdline and realpath (on musl) to
* work: */
if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
err (EXIT_FAILURE, "failed to mount proc on /proc");
we_mounted_proc = 1;
}
root_mountpoint = realpath (root_arg, NULL);
if (root_mountpoint == NULL)
err (EXIT_FAILURE, "realpath(\"%s\")", root_arg);
deploy_path = resolve_deploy_path (root_mountpoint);
if (we_mounted_proc)
{
/* Leave the filesystem in the state that we found it: */
if (umount ("/proc"))
err (EXIT_FAILURE, "failed to umount proc from /proc");
}
/* Work-around for a kernel bug: for some reason the kernel
* refuses switching root if any file systems are mounted
* MS_SHARED. Hence remount them MS_PRIVATE here as a

View File

@ -392,9 +392,9 @@ else
$OSTREE checkout test2 test2-checkout
fi
stat '--format=%Y' test2-checkout/baz/cow > cow-mtime
assert_file_has_content cow-mtime 1
assert_file_has_content cow-mtime 0
stat '--format=%Y' test2-checkout/baz/deeper > deeper-mtime
assert_file_has_content deeper-mtime 1
assert_file_has_content deeper-mtime 0
echo "ok content mtime"
cd ${test_tmpdir}

View File

@ -420,6 +420,10 @@ skip_without_fuse () {
[ -e /etc/mtab ] || skip "no /etc/mtab"
}
has_gpgme () {
${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'
}
libtest_cleanup_gpg () {
gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye || true
}

View File

@ -19,13 +19,13 @@
set -euo pipefail
if ! ostree --version | grep -q -e '\+gpgme'; then
. $(dirname $0)/libtest.sh
if ! has_gpgme; then
echo "1..0 #SKIP no gpg support compiled in"
exit 0
fi
. $(dirname $0)/libtest.sh
echo "1..1"
keyid="472CDAFA"

View File

@ -20,13 +20,13 @@
set -euo pipefail
if ! ostree --version | grep -q -e '\+gpgme'; then
. $(dirname $0)/libtest.sh
if ! has_gpgme; then
echo "1..0 #SKIP no gpgme support compiled in"
exit 0
fi
. $(dirname $0)/libtest.sh
echo "1..1"
setup_test_repository "archive-z2"

75
tests/test-pull-contenturl.sh Executable file
View File

@ -0,0 +1,75 @@
#!/bin/bash
#
# Copyright (C) 2016 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -euo pipefail
. $(dirname $0)/libtest.sh
echo "1..2"
COMMIT_SIGN=""
if has_gpgme; then
COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}"
fi
setup_fake_remote_repo1 "archive-z2" "${COMMIT_SIGN}"
# create a summary
${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo \
summary -u ${COMMIT_SIGN}
# Let's bring up an identical server in which meta files are missing
cd ${test_tmpdir}
mkdir httpd-content
cd httpd-content
cp -a ${test_tmpdir}/ostree-srv ostree
# delete all the meta stuff from here
rm ostree/gnomerepo/summary
if has_gpgme; then
rm ostree/gnomerepo/summary.sig
find ostree/gnomerepo/objects -name '*.commitmeta' | xargs rm
fi
# delete all the content stuff from there
find ${test_tmpdir}/ostree-srv/gnomerepo/objects \
! -name '*.commitmeta' -type f | xargs rm
${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
-p ${test_tmpdir}/httpd-content-port
content_port=$(cat ${test_tmpdir}/httpd-content-port)
echo "http://127.0.0.1:${content_port}" > ${test_tmpdir}/httpd-content-address
cd ${test_tmpdir}
mkdir repo
${CMD_PREFIX} ostree --repo=repo init
if has_gpgme; then VERIFY=true; else VERIFY=false; fi
${CMD_PREFIX} ostree --repo=repo remote add origin \
--set=gpg-verify=$VERIFY --set=gpg-verify-summary=$VERIFY \
--contenturl=$(cat httpd-content-address)/ostree/gnomerepo \
$(cat httpd-address)/ostree/gnomerepo
${CMD_PREFIX} ostree --repo=repo pull origin:main
echo "ok pull objects from contenturl"
if ! has_gpgme; then
echo "ok don't pull sigs from contenturl # SKIP not compiled with gpgme"
else
echo "ok don't pull sigs from contenturl"
fi

View File

@ -61,7 +61,7 @@ find repo/objects -name '*.filez' | while read name; do
done
echo "ok pull mirror summary"
if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then
if ! has_gpgme; then
exit 0;
fi

106
tests/test-pull-mirrorlist.sh Executable file
View File

@ -0,0 +1,106 @@
#!/bin/bash
#
# Copyright (C) 2016 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -euo pipefail
. $(dirname $0)/libtest.sh
echo "1..3"
setup_fake_remote_repo1 "archive-z2"
setup_mirror () {
name=$1; shift
cd ${test_tmpdir}
mkdir $name
cd $name
cp -a ${test_tmpdir}/ostree-srv ostree
${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
-p ${test_tmpdir}/${name}-port
port=$(cat ${test_tmpdir}/${name}-port)
echo "http://127.0.0.1:${port}" > ${test_tmpdir}/${name}-address
}
setup_mirror content_mirror1
setup_mirror content_mirror2
setup_mirror content_mirror3
# Let's delete a file from 1 so that it falls back on 2
cd ${test_tmpdir}/content_mirror1/ostree/gnomerepo
filez=$(find objects/ -name '*.filez' | head -n 1)
rm ${filez}
# Let's delete a file from 1 and 2 so that it falls back on 3
cd ${test_tmpdir}/content_mirror1/ostree/gnomerepo
filez=$(find objects/ -name '*.filez' | head -n 1)
rm ${filez}
cd ${test_tmpdir}/content_mirror2/ostree/gnomerepo
rm ${filez}
# OK, let's just shove the mirrorlist in the first httpd
cat > ${test_tmpdir}/ostree-srv/mirrorlist <<EOF
# comment with empty lines around
http://example.com/nonexistent
$(cat ${test_tmpdir}/content_mirror1-address)/ostree/gnomerepo
$(cat ${test_tmpdir}/content_mirror2-address)/ostree/gnomerepo
$(cat ${test_tmpdir}/content_mirror3-address)/ostree/gnomerepo
EOF
# first let's try just url
cd ${test_tmpdir}
mkdir repo
${CMD_PREFIX} ostree --repo=repo init
${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \
mirrorlist=$(cat httpd-address)/ostree/mirrorlist
${CMD_PREFIX} ostree --repo=repo pull origin:main
echo "ok pull objects from mirrorlist"
# now let's try contenturl only mirrorlist
cd ${test_tmpdir}
rm -rf repo
mkdir repo
${CMD_PREFIX} ostree --repo=repo init
${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \
--contenturl=mirrorlist=$(cat httpd-address)/ostree/mirrorlist \
$(cat httpd-address)/ostree/gnomerepo
${CMD_PREFIX} ostree --repo=repo pull origin:main
echo "ok pull objects from contenturl mirrorlist"
# both
cd ${test_tmpdir}
rm -rf repo
mkdir repo
${CMD_PREFIX} ostree --repo=repo init
${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \
--contenturl=mirrorlist=$(cat httpd-address)/ostree/mirrorlist \
mirrorlist=$(cat httpd-address)/ostree/mirrorlist
${CMD_PREFIX} ostree --repo=repo pull origin:main
echo "ok pull objects from split urls mirrorlists"

View File

@ -54,7 +54,7 @@ assert_file_has_content yet-another-copy/yet-another-hello-world "hello world ye
${CMD_PREFIX} ostree --repo=repo fsck
echo "ok pull mirror summary"
if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then
if ! has_gpgme; then
exit 0;
fi

View File

@ -4,9 +4,16 @@ this_script="${BASH_SOURCE:-$(readlink -f "$0")}"
setup_bootfs() {
mkdir -p "$1/proc" "$1/bin"
echo "quiet ostree=/ostree/boot.0 ro" >"$1/proc/cmdline"
# We need the real /proc mounted here so musl's realpath will work, but we
# want to be able to override /proc/cmdline, so bind mount.
mount -t proc proc "$1/proc"
echo "quiet ostree=/ostree/boot.0 ro" >"$1/override_cmdline"
mount --bind "$1/override_cmdline" "$1/proc/cmdline"
touch "$1/this_is_bootfs"
cp "$(dirname "$this_script")/../src/switchroot/ostree-prepare-root" "$1/bin"
cp "$(dirname "$this_script")/../ostree-prepare-root" "$1/bin"
}
setup_rootfs() {