diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 44b6cd0a..6214633d 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -42,6 +42,7 @@ if ENABLE_EXPERIMENTAL_API libostree_public_headers += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ + src/libostree/ostree-repo-finder.h \ $(NULL) endif diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a9c392c1..a9331dd4 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -154,6 +154,11 @@ if !ENABLE_EXPERIMENTAL_API libostree_1_la_SOURCES += \ src/libostree/ostree-ref.h \ src/libostree/ostree-remote.h \ + src/libostree/ostree-repo-finder.h \ + $(NULL) +else # if ENABLE_EXPERIMENTAL_API +libostree_1_la_SOURCES += \ + src/libostree/ostree-repo-finder.c \ $(NULL) endif @@ -230,7 +235,7 @@ OSTree_1_0_gir_INCLUDES = Gio-2.0 OSTree_1_0_gir_CFLAGS = $(libostree_1_la_CFLAGS) OSTree_1_0_gir_LIBS = libostree-1.la OSTree_1_0_gir_SCANNERFLAGS = --warn-all --identifier-prefix=Ostree --symbol-prefix=ostree -OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h,$(libostree_1_la_SOURCES)) +OSTree_1_0_gir_FILES = $(libostreeinclude_HEADERS) $(filter-out %-private.h %/ostree-soup-uri.h %/ostree-repo-finder.h,$(libostree_1_la_SOURCES)) INTROSPECTION_GIRS += OSTree-1.0.gir gir_DATA += OSTree-1.0.gir typelib_DATA += OSTree-1.0.typelib diff --git a/apidoc/ostree-experimental-sections.txt b/apidoc/ostree-experimental-sections.txt index 78a50100..7e9ed084 100644 --- a/apidoc/ostree-experimental-sections.txt +++ b/apidoc/ostree-experimental-sections.txt @@ -21,6 +21,34 @@ ostree_remote_unref ostree_remote_get_name +
+ostree-repo-experimental +ostree_repo_find_remotes_async +ostree_repo_find_remotes_finish +ostree_repo_pull_from_remotes_async +ostree_repo_pull_from_remotes_finish +ostree_repo_resolve_keyring_for_collection +
+ +
+ostree-repo-finder +OstreeRepoFinder +ostree_repo_finder_resolve_async +ostree_repo_finder_resolve_finish +ostree_repo_finder_resolve_all_async +ostree_repo_finder_resolve_all_finish +OstreeRepoFinderResult +ostree_repo_finder_result_new +ostree_repo_finder_result_dup +ostree_repo_finder_result_free +ostree_repo_finder_result_compare +OstreeRepoFinderResultv +ostree_repo_finder_result_freev + +ostree_repo_finder_get_type +ostree_repo_finder_result_get_type +
+
ostree-misc-experimental ostree_repo_get_collection_id diff --git a/src/libostree/libostree-experimental.sym b/src/libostree/libostree-experimental.sym index 9d2024f3..cda34322 100644 --- a/src/libostree/libostree-experimental.sym +++ b/src/libostree/libostree-experimental.sym @@ -45,8 +45,24 @@ global: ostree_collection_ref_get_type; ostree_collection_ref_hash; ostree_collection_ref_new; + ostree_repo_find_remotes_async; + ostree_repo_find_remotes_finish; + ostree_repo_finder_get_type; + ostree_repo_finder_resolve_async; + ostree_repo_finder_resolve_all_async; + ostree_repo_finder_resolve_all_finish; + ostree_repo_finder_resolve_finish; + ostree_repo_finder_result_compare; + ostree_repo_finder_result_dup; + ostree_repo_finder_result_free; + ostree_repo_finder_result_freev; + ostree_repo_finder_result_get_type; + ostree_repo_finder_result_new; ostree_repo_get_collection_id; ostree_repo_list_collection_refs; + ostree_repo_pull_from_remotes_async; + ostree_repo_pull_from_remotes_finish; + ostree_repo_resolve_keyring_for_collection; ostree_repo_set_collection_id; ostree_repo_set_collection_ref_immediate; ostree_repo_transaction_set_collection_ref; diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index f683c2e3..1f7716b2 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -63,6 +63,9 @@ G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_comm G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #endif diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index a4a31034..a8fbb8e1 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -182,6 +182,11 @@ gboolean ostree_validate_collection_id (const char *collection_id, GError **erro #include "ostree-ref.h" G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL) + +#include "ostree-repo-finder.h" +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) +G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) #endif G_END_DECLS diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c new file mode 100644 index 00000000..7893978d --- /dev/null +++ b/src/libostree/ostree-repo-finder.c @@ -0,0 +1,576 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "ostree-autocleanups.h" +#include "ostree-core.h" +#include "ostree-remote-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo.h" + +static void ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface); + +G_DEFINE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, G_TYPE_OBJECT) + +static void +ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface) +{ + /* Nothing to see here. */ +} + +/* Validate the given struct contains a valid collection ID and ref name, and that + * the collection ID is non-%NULL. */ +static gboolean +is_valid_collection_ref (const OstreeCollectionRef *ref) +{ + return (ref != NULL && + ostree_validate_rev (ref->ref_name, NULL) && + ostree_validate_collection_id (ref->collection_id, NULL)); +} + +/* Validate @refs is non-%NULL, non-empty, and contains only valid collection + * and ref names. */ +static gboolean +is_valid_collection_ref_array (const OstreeCollectionRef * const *refs) +{ + gsize i; + + if (refs == NULL || *refs == NULL) + return FALSE; + + for (i = 0; refs[i] != NULL; i++) + { + if (!is_valid_collection_ref (refs[i])) + return FALSE; + } + + return TRUE; +} + +/* Validate @ref_to_checksum is non-%NULL, non-empty, and contains only valid + * OstreeCollectionRefs as keys and only valid commit checksums as values. */ +static gboolean +is_valid_collection_ref_map (GHashTable *ref_to_checksum) +{ + GHashTableIter iter; + const OstreeCollectionRef *ref; + const gchar *checksum; + + if (ref_to_checksum == NULL || g_hash_table_size (ref_to_checksum) == 0) + return FALSE; + + g_hash_table_iter_init (&iter, ref_to_checksum); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + { + g_assert (ref != NULL); + g_assert (checksum != NULL); + + if (!is_valid_collection_ref (ref)) + return FALSE; + if (!ostree_validate_checksum_string (checksum, NULL)) + return FALSE; + } + + return TRUE; +} + +static void resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); + +/** + * ostree_repo_finder_resolve_async: + * @self: an #OstreeRepoFinder + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @parent_repo: (transfer none): the local repository which the refs are being resolved for, + * which provides configuration information and GPG keys + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Find reachable remote URIs which claim to provide any of the given @refs. The + * specific method for finding the remotes depends on the #OstreeRepoFinder + * implementation. + * + * Any remote which is found and which claims to support any of the given @refs + * will be returned in the results. It is possible that a remote claims to + * support a given ref, but turns out not to — it is not possible to verify this + * until ostree_repo_pull_from_remotes_async() is called. + * + * The returned results will be sorted with the most useful first — this is + * typically the remote which claims to provide the most @refs, at the lowest + * latency. + * + * Each result contains a mapping of @refs to the checksums of the commits + * which the result provides. If the result provides the latest commit for a ref + * across all of the results, the checksum will be set. Otherwise, if the + * result provides an outdated commit, or doesn’t provide a given ref at all, + * the ref will not be set. Results which provide none of the requested @refs + * may be listed with an empty refs map. + * + * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given + * @refs from those remotes. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + OstreeRepoFinder *finders[2] = { NULL, }; + + g_return_if_fail (OSTREE_IS_REPO_FINDER (self)); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (OSTREE_IS_REPO (parent_repo)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_async); + + finders[0] = self; + + ostree_repo_finder_resolve_all_async (finders, refs, parent_repo, cancellable, + resolve_cb, g_steal_pointer (&task)); +} + +static void +resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + + task = G_TASK (user_data); + + results = ostree_repo_finder_resolve_all_finish (result, &local_error); + + g_assert ((local_error == NULL) != (results == NULL)); + + if (local_error != NULL) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +/** + * ostree_repo_finder_resolve_finish: + * @self: an #OstreeRepoFinder + * @result: #GAsyncResult from the callback + * @error: return location for a #GError + * + * Get the results from a ostree_repo_finder_resolve_async() operation. + * + * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero + * or more results + * Since: 2017.8 + */ +GPtrArray * +ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO_FINDER (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +static gint +sort_results_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +typedef struct +{ + gsize n_finders_pending; + GPtrArray *results; +} ResolveAllData; + +static void +resolve_all_data_free (ResolveAllData *data) +{ + g_assert (data->n_finders_pending == 0); + g_clear_pointer (&data->results, g_ptr_array_unref); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ResolveAllData, resolve_all_data_free) + +static void resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); +static void resolve_all_finished_one (GTask *task); + +/** + * ostree_repo_finder_resolve_all_async: + * @finders: (array zero-terminated=1): non-empty array of #OstreeRepoFinders + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @parent_repo: (transfer none): the local repository which the refs are being resolved for, + * which provides configuration information and GPG keys + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * A version of ostree_repo_finder_resolve_async() which queries one or more + * @finders in parallel and combines the results. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(ResolveAllData) data = NULL; + gsize i; + g_autoptr(GString) refs_str = NULL; + g_autoptr(GString) finders_str = NULL; + + g_return_if_fail (finders != NULL && finders[0] != NULL); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (OSTREE_IS_REPO (parent_repo)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + refs_str = g_string_new (""); + for (i = 0; refs[i] != NULL; i++) + { + if (i != 0) + g_string_append (refs_str, ", "); + g_string_append_printf (refs_str, "(%s, %s)", + refs[i]->collection_id, refs[i]->ref_name); + } + + finders_str = g_string_new (""); + for (i = 0; finders[i] != NULL; i++) + { + if (i != 0) + g_string_append (finders_str, ", "); + g_string_append (finders_str, g_type_name (G_TYPE_FROM_INSTANCE (finders[i]))); + } + + g_debug ("%s: Resolving refs [%s] with finders [%s]", G_STRFUNC, + refs_str->str, finders_str->str); + + task = g_task_new (NULL, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_all_async); + + data = g_new0 (ResolveAllData, 1); + data->n_finders_pending = 1; /* while setting up the loop */ + data->results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + g_task_set_task_data (task, data, (GDestroyNotify) resolve_all_data_free); + + /* Start all the asynchronous queries in parallel. */ + for (i = 0; finders[i] != NULL; i++) + { + OstreeRepoFinder *finder = OSTREE_REPO_FINDER (finders[i]); + OstreeRepoFinderInterface *iface; + + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + g_assert (iface->resolve_async != NULL); + iface->resolve_async (finder, refs, parent_repo, cancellable, resolve_all_cb, g_object_ref (task)); + data->n_finders_pending++; + } + + resolve_all_finished_one (task); + data = NULL; /* passed to the GTask above */ +} + +/* Modifies both arrays in place. */ +static void +array_concatenate_steal (GPtrArray *array, + GPtrArray *to_concatenate) /* (transfer full) */ +{ + g_autoptr(GPtrArray) array_to_concatenate = to_concatenate; + gsize i; + + for (i = 0; i < array_to_concatenate->len; i++) + { + /* Sanity check that the arrays do not contain any %NULL elements + * (particularly NULL terminators). */ + g_assert (g_ptr_array_index (array_to_concatenate, i) != NULL); + g_ptr_array_add (array, g_steal_pointer (&g_ptr_array_index (array_to_concatenate, i))); + } + + g_ptr_array_set_free_func (array_to_concatenate, NULL); + g_ptr_array_set_size (array_to_concatenate, 0); +} + +static void +resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepoFinder *finder; + OstreeRepoFinderInterface *iface; + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + ResolveAllData *data; + + finder = OSTREE_REPO_FINDER (obj); + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + task = G_TASK (user_data); + data = g_task_get_task_data (task); + results = iface->resolve_finish (finder, result, &local_error); + + g_assert ((local_error == NULL) != (results == NULL)); + + if (local_error != NULL) + g_debug ("Error resolving refs to repository URI using %s: %s", + g_type_name (G_TYPE_FROM_INSTANCE (finder)), local_error->message); + else + array_concatenate_steal (data->results, g_steal_pointer (&results)); + + resolve_all_finished_one (task); +} + +static void +resolve_all_finished_one (GTask *task) +{ + ResolveAllData *data; + + data = g_task_get_task_data (task); + + data->n_finders_pending--; + + if (data->n_finders_pending == 0) + { + gsize i; + g_autoptr(GString) results_str = NULL; + + g_ptr_array_sort (data->results, sort_results_cb); + + results_str = g_string_new (""); + for (i = 0; i < data->results->len; i++) + { + const OstreeRepoFinderResult *result = g_ptr_array_index (data->results, i); + + if (i != 0) + g_string_append (results_str, ", "); + g_string_append (results_str, ostree_remote_get_name (result->remote)); + } + if (i == 0) + g_string_append (results_str, "(none)"); + + g_debug ("%s: Finished, results: %s", G_STRFUNC, results_str->str); + + g_task_return_pointer (task, g_steal_pointer (&data->results), (GDestroyNotify) g_ptr_array_unref); + } +} + +/** + * ostree_repo_finder_resolve_all_finish: + * @result: #GAsyncResult from the callback + * @error: return location for a #GError + * + * Get the results from a ostree_repo_finder_resolve_all_async() operation. + * + * Returns: (transfer full) (element-type OstreeRepoFinderResult): array of zero + * or more results + * Since: 2017.8 + */ +GPtrArray * +ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, NULL), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +G_DEFINE_BOXED_TYPE (OstreeRepoFinderResult, ostree_repo_finder_result, + ostree_repo_finder_result_dup, ostree_repo_finder_result_free) + +/** + * ostree_repo_finder_result_new: + * @remote: (transfer none): an #OstreeRemote containing the transport details + * for the result + * @finder: (transfer none): the #OstreeRepoFinder instance which produced the + * result + * @priority: static priority of the result, where higher numbers indicate lower + * priority + * @ref_to_checksum: (element-type OstreeCollectionRef utf8): map of collection–ref pairs + * to checksums provided by this result + * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when + * the summary file for the result was last modified, or `0` if this is unknown + * + * Create a new #OstreeRepoFinderResult instance. The semantics for the arguments + * are as described in the #OstreeRepoFinderResult documentation. + * + * Returns: (transfer full): a new #OstreeRepoFinderResult + * Since: 2017.8 + */ +OstreeRepoFinderResult * +ostree_repo_finder_result_new (OstreeRemote *remote, + OstreeRepoFinder *finder, + gint priority, + GHashTable *ref_to_checksum, + guint64 summary_last_modified) +{ + g_autoptr(OstreeRepoFinderResult) result = NULL; + + g_return_val_if_fail (remote != NULL, NULL); + g_return_val_if_fail (OSTREE_IS_REPO_FINDER (finder), NULL); + g_return_val_if_fail (is_valid_collection_ref_map (ref_to_checksum), NULL); + + result = g_new0 (OstreeRepoFinderResult, 1); + result->remote = ostree_remote_ref (remote); + result->finder = g_object_ref (finder); + result->priority = priority; + result->ref_to_checksum = g_hash_table_ref (ref_to_checksum); + result->summary_last_modified = summary_last_modified; + + return g_steal_pointer (&result); +} + +/** + * ostree_repo_finder_result_dup: + * @result: (transfer none): an #OstreeRepoFinderResult to copy + * + * Copy an #OstreeRepoFinderResult. + * + * Returns: (transfer full): a newly allocated copy of @result + * Since: 2017.8 + */ +OstreeRepoFinderResult * +ostree_repo_finder_result_dup (OstreeRepoFinderResult *result) +{ + g_return_val_if_fail (result != NULL, NULL); + + return ostree_repo_finder_result_new (result->remote, result->finder, + result->priority, result->ref_to_checksum, + result->summary_last_modified); +} + +/** + * ostree_repo_finder_result_compare: + * @a: an #OstreeRepoFinderResult + * @b: an #OstreeRepoFinderResult + * + * Compare two #OstreeRepoFinderResult instances to work out which one is better + * to pull from, and hence needs to be ordered before the other. + * + * Returns: <0 if @a is ordered before @b, 0 if they are ordered equally, + * >0 if @b is ordered before @a + * Since: 2017.8 + */ +gint +ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b) +{ + guint a_n_refs, b_n_refs; + + g_return_val_if_fail (a != NULL, 0); + g_return_val_if_fail (b != NULL, 0); + + /* FIXME: Check if this is really the ordering we want. For example, we + * probably don’t want a result with 0 refs to be ordered before one with >0 + * refs, just because its priority is higher. */ + if (a->priority != b->priority) + return a->priority - b->priority; + + if (a->summary_last_modified != 0 && b->summary_last_modified != 0 && + a->summary_last_modified != b->summary_last_modified) + return a->summary_last_modified - b->summary_last_modified; + + gpointer value; + GHashTableIter iter; + a_n_refs = b_n_refs = 0; + + g_hash_table_iter_init (&iter, a->ref_to_checksum); + while (g_hash_table_iter_next (&iter, NULL, &value)) + if (value != NULL) + a_n_refs++; + + g_hash_table_iter_init (&iter, b->ref_to_checksum); + while (g_hash_table_iter_next (&iter, NULL, &value)) + if (value != NULL) + b_n_refs++; + + if (a_n_refs != b_n_refs) + return (gint) a_n_refs - (gint) b_n_refs; + + return g_strcmp0 (a->remote->name, b->remote->name); +} + +/** + * ostree_repo_finder_result_free: + * @result: (transfer full): an #OstreeRepoFinderResult + * + * Free the given @result. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_result_free (OstreeRepoFinderResult *result) +{ + g_return_if_fail (result != NULL); + + g_hash_table_unref (result->ref_to_checksum); + g_object_unref (result->finder); + ostree_remote_unref (result->remote); + g_free (result); +} + +/** + * ostree_repo_finder_result_freev: + * @results: (array zero-terminated=1) (transfer full): an #OstreeRepoFinderResult + * + * Free the given @results array, freeing each element and the container. + * + * Since: 2017.8 + */ +void +ostree_repo_finder_result_freev (OstreeRepoFinderResult **results) +{ + gsize i; + + for (i = 0; results[i] != NULL; i++) + ostree_repo_finder_result_free (results[i]); + + g_free (results); +} diff --git a/src/libostree/ostree-repo-finder.h b/src/libostree/ostree-repo-finder.h new file mode 100644 index 00000000..6b0ce8ca --- /dev/null +++ b/src/libostree/ostree-repo-finder.h @@ -0,0 +1,172 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-ref.h" +#include "ostree-remote.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER (ostree_repo_finder_get_type ()) + +/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44): +_OSTREE_PUBLIC +G_DECLARE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, OSTREE, REPO_FINDER, GObject) */ + +_OSTREE_PUBLIC +GType ostree_repo_finder_get_type (void); +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeRepoFinder OstreeRepoFinder; +typedef struct _OstreeRepoFinderInterface OstreeRepoFinderInterface; + +static inline OstreeRepoFinder *OSTREE_REPO_FINDER (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_get_type (), OstreeRepoFinder); } +static inline gboolean OSTREE_IS_REPO_FINDER (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_get_type ()); } +static inline OstreeRepoFinderInterface *OSTREE_REPO_FINDER_GET_IFACE (gpointer ptr) { return G_TYPE_INSTANCE_GET_INTERFACE (ptr, ostree_repo_finder_get_type (), OstreeRepoFinderInterface); } +G_GNUC_END_IGNORE_DEPRECATIONS + +struct _OstreeRepoFinderInterface +{ + GTypeInterface g_iface; + + void (*resolve_async) (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + GPtrArray *(*resolve_finish) (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); +}; + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const OstreeCollectionRef * const *refs, + OstreeRepo *parent_repo, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error); + +/** + * OstreeRepoFinderResult: + * @remote: #OstreeRemote which contains the transport details for the result, + * such as its URI and GPG key + * @finder: the #OstreeRepoFinder instance which produced this result + * @priority: static priority of the result, where higher numbers indicate lower + * priority + * @ref_to_checksum: (element-type OstreeCollectionRef utf8): map of collection–ref + * pairs to checksums provided by this remote; values may be %NULL to + * indicate this remote doesn’t provide that ref + * @summary_last_modified: Unix timestamp (seconds since the epoch, UTC) when + * the summary file on the remote was last modified, or `0` if unknown + * + * #OstreeRepoFinderResult gives a single result from an + * ostree_repo_finder_resolve_async() or ostree_repo_finder_resolve_all_async() + * operation. This represents a single remote which provides none, some or all + * of the refs being resolved. The structure includes various bits of metadata + * which allow ostree_repo_pull_from_remotes_async() (for example) to prioritise + * how to pull the refs. + * + * The @priority is used as one input of many to ordering functions like + * ostree_repo_finder_result_compare(). + * + * @ref_to_checksum indicates which refs (out of the ones queried for as inputs + * to ostree_repo_finder_resolve_async()) are provided by this remote. The refs + * are present as keys (of type #OstreeCollectionRef), and the corresponding values + * are the checksums of the commits the remote currently has for those refs. (These + * might not be the latest commits available out of all results.) A + * checksum may be %NULL if the remote does not advertise the corresponding ref. + * After ostree_repo_finder_resolve_async() has been called, the commit metadata + * should be available locally, so the details for each checksum can be looked + * up using ostree_repo_load_commit(). + * + * Since: 2017.8 + */ +typedef struct +{ + OstreeRemote *remote; + OstreeRepoFinder *finder; + gint priority; + GHashTable *ref_to_checksum; + guint64 summary_last_modified; + + /*< private >*/ + gpointer padding[4]; +} OstreeRepoFinderResult; + +_OSTREE_PUBLIC +GType ostree_repo_finder_result_get_type (void); + +_OSTREE_PUBLIC +OstreeRepoFinderResult *ostree_repo_finder_result_new (OstreeRemote *remote, + OstreeRepoFinder *finder, + gint priority, + GHashTable *ref_to_checksum, + guint64 summary_last_modified); +_OSTREE_PUBLIC +OstreeRepoFinderResult *ostree_repo_finder_result_dup (OstreeRepoFinderResult *result); +_OSTREE_PUBLIC +gint ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b); +_OSTREE_PUBLIC +void ostree_repo_finder_result_free (OstreeRepoFinderResult *result); + +/** + * OstreeRepoFinderResultv: + * + * A %NULL-terminated array of #OstreeRepoFinderResult instances, designed to + * be used with g_auto(): + * + * |[ + * g_auto(OstreeRepoFinderResultv) results = NULL; + * ]| + * + * Since: 2017.8 + */ +typedef OstreeRepoFinderResult** OstreeRepoFinderResultv; + +_OSTREE_PUBLIC +void ostree_repo_finder_result_freev (OstreeRepoFinderResult **results); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 03be117a..b4c565a8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1,6 +1,7 @@ /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- * * Copyright (C) 2011,2012,2013 Colin Walters + * Copyright © 2017 Endless Mobile, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -17,7 +18,9 @@ * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. * - * Author: Colin Walters + * Authors: + * - Colin Walters + * - Philip Withnall */ #include "config.h" @@ -33,8 +36,13 @@ #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" #include "ostree-fetcher-util.h" +#include "ostree-remote-private.h" #include "ot-fs-utils.h" +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API +#include "ostree-repo-finder.h" +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + #include #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) @@ -2611,6 +2619,11 @@ repo_remote_fetch_summary (OstreeRepo *self, } } + /* FIXME: Send the ETag from the cache with the request for summary.sig to + * avoid downloading summary.sig unnecessarily. This won’t normally provide + * any benefits (but won’t do any harm) since summary.sig is typically 500B + * in size. But if a repository has multiple keys, the signature file will + * grow and this optimisation may be useful. */ if (!_ostree_preload_metadata_file (self, fetcher, mirrorlist, @@ -3786,6 +3799,1302 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } +#ifdef OSTREE_ENABLE_EXPERIMENTAL_API + +/* Structure used in ostree_repo_find_remotes_async() which stores metadata + * about a given OSTree commit. This includes the metadata from the commit + * #GVariant, plus some working state which is used to work out which remotes + * have refs pointing to this commit. */ +typedef struct +{ + gchar *checksum; /* always set */ + guint64 commit_size; /* always set */ + guint64 timestamp; /* 0 for unknown */ + GVariant *additional_metadata; + GArray *refs; /* (element-type gsize), indexes to refs which point to this commit on at least one remote */ +} CommitMetadata; + +static void +commit_metadata_free (CommitMetadata *info) +{ + g_clear_pointer (&info->refs, g_array_unref); + g_free (info->checksum); + g_clear_pointer (&info->additional_metadata, g_variant_unref); + g_free (info); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitMetadata, commit_metadata_free) + +static CommitMetadata * +commit_metadata_new (const gchar *checksum, + guint64 commit_size, + guint64 timestamp, + GVariant *additional_metadata) +{ + g_autoptr(CommitMetadata) info = NULL; + + info = g_new0 (CommitMetadata, 1); + info->checksum = g_strdup (checksum); + info->commit_size = commit_size; + info->timestamp = timestamp; + info->additional_metadata = (additional_metadata != NULL) ? g_variant_ref (additional_metadata) : NULL; + info->refs = g_array_new (FALSE, FALSE, sizeof (gsize)); + + return g_steal_pointer (&info); +} + +/* Structure used in ostree_repo_find_remotes_async() to store a grid (or table) + * of pointers, indexed by rows and columns. Basically an encapsulated 2D array. + * See the comments in ostree_repo_find_remotes_async() for its semantics + * there. */ +typedef struct +{ + gsize width; /* pointers */ + gsize height; /* pointers */ + gconstpointer pointers[]; /* n_pointers = width * height */ +} PointerTable; + +static void +pointer_table_free (PointerTable *table) +{ + g_free (table); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (PointerTable, pointer_table_free) + +/* Both dimensions are in numbers of pointers. */ +static PointerTable * +pointer_table_new (gsize width, + gsize height) +{ + g_autoptr(PointerTable) table = NULL; + + g_return_val_if_fail (width > 0, NULL); + g_return_val_if_fail (height > 0, NULL); + g_return_val_if_fail (width <= (G_MAXSIZE - sizeof (PointerTable)) / sizeof (gconstpointer) / height, NULL); + + table = g_malloc0 (sizeof (PointerTable) + sizeof (gconstpointer) * width * height); + table->width = width; + table->height = height; + + return g_steal_pointer (&table); +} + +static gconstpointer +pointer_table_get (const PointerTable *table, + gsize x, + gsize y) +{ + g_return_val_if_fail (table != NULL, FALSE); + g_return_val_if_fail (x < table->width, FALSE); + g_return_val_if_fail (y < table->height, FALSE); + + return table->pointers[table->width * y + x]; +} + +static void +pointer_table_set (PointerTable *table, + gsize x, + gsize y, + gconstpointer value) +{ + g_return_if_fail (table != NULL); + g_return_if_fail (x < table->width); + g_return_if_fail (y < table->height); + + table->pointers[table->width * y + x] = value; +} + +/* Validate the given struct contains a valid collection ID and ref name. */ +static gboolean +is_valid_collection_ref (const OstreeCollectionRef *ref) +{ + return (ref != NULL && + ostree_validate_rev (ref->ref_name, NULL) && + ostree_validate_collection_id (ref->collection_id, NULL)); +} + +/* Validate @refs is non-%NULL, non-empty, and contains only valid collection + * and ref names. */ +static gboolean +is_valid_collection_ref_array (const OstreeCollectionRef * const *refs) +{ + gsize i; + + if (refs == NULL || *refs == NULL) + return FALSE; + + for (i = 0; refs[i] != NULL; i++) + { + if (!is_valid_collection_ref (refs[i])) + return FALSE; + } + + return TRUE; +} + +/* Validate @finders is non-%NULL, non-empty, and contains only valid + * #OstreeRepoFinder instances. */ +static gboolean +is_valid_finder_array (OstreeRepoFinder **finders) +{ + gsize i; + + if (finders == NULL || *finders == NULL) + return FALSE; + + for (i = 0; finders[i] != NULL; i++) + { + if (!OSTREE_IS_REPO_FINDER (finders[i])) + return FALSE; + } + + return TRUE; +} + +/* Closure used to carry inputs from ostree_repo_find_remotes_async() to + * find_remotes_cb(). */ +typedef struct +{ + OstreeCollectionRef **refs; + GVariant *options; + OstreeAsyncProgress *progress; +} FindRemotesData; + +static void +find_remotes_data_free (FindRemotesData *data) +{ + g_clear_object (&data->progress); + g_clear_pointer (&data->options, g_variant_unref); + ostree_collection_ref_freev (data->refs); + + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (FindRemotesData, find_remotes_data_free) + +static FindRemotesData * +find_remotes_data_new (const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeAsyncProgress *progress) +{ + g_autoptr(FindRemotesData) data = NULL; + + data = g_new0 (FindRemotesData, 1); + data->refs = ostree_collection_ref_dupv (refs); + data->options = (options != NULL) ? g_variant_ref (options) : NULL; + data->progress = (progress != NULL) ? g_object_ref (progress) : NULL; + + return g_steal_pointer (&data); +} + +static gchar * +uint64_secs_to_iso8601 (guint64 secs) +{ + g_autoptr(GDateTime) dt = g_date_time_new_from_unix_utc (secs); + + if (dt != NULL) + return g_date_time_format (dt, "%FT%TZ"); + else + return g_strdup ("invalid"); +} + +static gint +sort_results_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult **result_a = (const OstreeRepoFinderResult **) a; + const OstreeRepoFinderResult **result_b = (const OstreeRepoFinderResult **) b; + + return ostree_repo_finder_result_compare (*result_a, *result_b); +} + +static void +repo_finder_result_free0 (OstreeRepoFinderResult *result) +{ + if (result == NULL) + return; + + ostree_repo_finder_result_free (result); +} + +static void find_remotes_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); + +/** + * ostree_repo_find_remotes_async: + * @self: an #OstreeRepo + * @refs: (array zero-terminated=1): non-empty array of collection–ref pairs to find remotes for + * @options: (nullable): a GVariant `a{sv}` with an extensible set of flags + * @finders: (array zero-terminated=1) (transfer none): non-empty array of + * #OstreeRepoFinder instances to use, or %NULL to use the system defaults + * @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s + * progress, or %NULL + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Find reachable remote URIs which claim to provide any of the given named + * @refs. This will search for configured remotes (#OstreeRepoFinderConfig), + * mounted volumes (#OstreeRepoFinderMount) and (if enabled at compile time) + * local network peers (#OstreeRepoFinderAvahi). In order to use a custom + * configuration of #OstreeRepoFinder instances, call + * ostree_repo_finder_resolve_all_async() on them individually. + * + * Any remote which is found and which claims to support any of the given @refs + * will be returned in the results. It is possible that a remote claims to + * support a given ref, but turns out not to — it is not possible to verify this + * until ostree_repo_pull_from_remotes_async() is called. + * + * The returned results will be sorted with the most useful first — this is + * typically the remote which claims to provide the most of @refs, at the lowest + * latency. + * + * Each result contains a list of the subset of @refs it claims to provide. It + * is possible for a non-empty list of results to be returned, but for some of + * @refs to not be listed in any of the results. Callers must check for this. + * + * Pass the results to ostree_repo_pull_from_remotes_async() to pull the given @refs + * from those remotes. + * + * No @options are currently supported. + * + * @finders must be a non-empty %NULL-terminated array of the #OstreeRepoFinder + * instances to use, or %NULL to use the system default set of finders, which + * will typically be all available finders using their default options (but + * this is not guaranteed). + * + * GPG verification of the summary and all commits will be used unconditionally. + * + * This will use the thread-default #GMainContext, but will not iterate it. + * + * Since: 2017.8 + */ +void +ostree_repo_find_remotes_async (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeRepoFinder **finders, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(FindRemotesData) data = NULL; + GMainContext *context; + OstreeRepoFinder *default_finders[4] = { NULL, }; + + g_return_if_fail (OSTREE_IS_REPO (self)); + g_return_if_fail (is_valid_collection_ref_array (refs)); + g_return_if_fail (options == NULL || + g_variant_is_of_type (options, G_VARIANT_TYPE_VARDICT)); + g_return_if_fail (finders == NULL || is_valid_finder_array (finders)); + g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + /* Set up a task for the whole operation. */ + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_find_remotes_async); + + context = g_main_context_get_thread_default (); + + /* Are we using #OstreeRepoFinders provided by the user, or the defaults? */ + if (finders == NULL) + { + finders = default_finders; + } + + data = find_remotes_data_new (refs, options, progress); + g_task_set_task_data (task, g_steal_pointer (&data), (GDestroyNotify) find_remotes_data_free); + + /* Asynchronously resolve all possible remotes for the given refs. */ + ostree_repo_finder_resolve_all_async (finders, refs, self, cancellable, + find_remotes_cb, g_steal_pointer (&task)); +} + +/* Find the first instance of (@collection_id, @ref_name) in @refs and return + * its index; or return %FALSE if nothing’s found. */ +static gboolean +collection_refv_contains (const OstreeCollectionRef * const *refs, + const gchar *collection_id, + const gchar *ref_name, + gsize *out_index) +{ + gsize i; + + for (i = 0; refs[i] != NULL; i++) + { + if (g_str_equal (refs[i]->collection_id, collection_id) && + g_str_equal (refs[i]->ref_name, ref_name)) + { + *out_index = i; + return TRUE; + } + } + + return FALSE; +} + +/* For each ref from @refs which is listed in @summary_refs, cache its metadata + * from the summary file entry into @commit_metadatas, and add the checksum it + * points to into @refs_and_remotes_table at (@ref_index, @result_index). + * @ref_index is the ref’s index in @refs. */ +static gboolean +find_remotes_process_refs (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + OstreeRepoFinderResult *result, + gsize result_index, + const gchar *summary_collection_id, + GVariant *summary_refs, + GHashTable *commit_metadatas, + PointerTable *refs_and_remotes_table) +{ + gsize j, n; + + for (j = 0, n = g_variant_n_children (summary_refs); j < n; j++) + { + const guchar *csum_bytes; + g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL, stored_commit_metadata_v = NULL; + guint64 commit_size, commit_timestamp; + gchar tmp_checksum[OSTREE_SHA256_STRING_LEN + 1]; + gsize ref_index; + g_autoptr(GDateTime) dt = NULL; + g_autoptr(GError) error = NULL; + const gchar *ref_name; + CommitMetadata *commit_metadata; + + /* Check the ref name. */ + ref_v = g_variant_get_child_value (summary_refs, j); + g_variant_get_child (ref_v, 0, "&s", &ref_name); + + if (!ostree_validate_rev (ref_name, &error)) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref name ‘%s’: %s", + G_STRFUNC, result->remote->name, ref_name, error->message); + return FALSE; + } + + /* Check the commit checksum. */ + g_variant_get_child (ref_v, 1, "(t@ay@a{sv})", &commit_size, &csum_v, &commit_metadata_v); + + csum_bytes = ostree_checksum_bytes_peek_validate (csum_v, &error); + if (csum_bytes == NULL) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref checksum: %s", + G_STRFUNC, result->remote->name, error->message); + return FALSE; + } + + ostree_checksum_inplace_from_bytes (csum_bytes, tmp_checksum); + + /* Is this a ref we care about? */ + if (!collection_refv_contains (refs, summary_collection_id, ref_name, &ref_index)) + continue; + + /* Load the commit metadata from disk if possible, for verification. */ + if (!ostree_repo_load_commit (self, tmp_checksum, &stored_commit_metadata_v, NULL, NULL)) + stored_commit_metadata_v = NULL; + + /* Check the additional metadata. */ + if (!g_variant_lookup (commit_metadata_v, OSTREE_COMMIT_TIMESTAMP, "t", &commit_timestamp)) + commit_timestamp = 0; /* unknown */ + else + commit_timestamp = GUINT64_FROM_BE (commit_timestamp); + + dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt == NULL) + { + g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.", + G_STRFUNC, result->remote->name, commit_timestamp); + commit_timestamp = 0; + } + + /* Check and store the commit metadata. */ + commit_metadata = g_hash_table_lookup (commit_metadatas, tmp_checksum); + + if (commit_metadata == NULL) + { + commit_metadata = commit_metadata_new (tmp_checksum, commit_size, + (stored_commit_metadata_v != NULL) ? ostree_commit_get_timestamp (stored_commit_metadata_v) : 0, + NULL); + g_hash_table_insert (commit_metadatas, commit_metadata->checksum, + commit_metadata /* transfer */); + } + + /* Update the metadata if possible. */ + if (commit_metadata->timestamp == 0) + { + commit_metadata->timestamp = commit_timestamp; + } + else if (commit_timestamp != 0 && commit_metadata->timestamp != commit_timestamp) + { + g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which did not match existing timestamp %" G_GUINT64_FORMAT ". Ignoring.", + G_STRFUNC, result->remote->name, commit_timestamp, commit_metadata->timestamp); + return FALSE; + } + + if (commit_size != commit_metadata->commit_size) + { + g_debug ("%s: Summary for result ‘%s’ contained commit size %" G_GUINT64_FORMAT "B which did not match existing size %" G_GUINT64_FORMAT "B. Ignoring.", + G_STRFUNC, result->remote->name, commit_size, commit_metadata->commit_size); + return FALSE; + } + + pointer_table_set (refs_and_remotes_table, ref_index, result_index, commit_metadata->checksum); + g_array_append_val (commit_metadata->refs, ref_index); + + g_debug ("%s: Remote ‘%s’ lists ref ‘%s’ mapping to commit ‘%s’.", + G_STRFUNC, result->remote->name, ref_name, commit_metadata->checksum); + } + + return TRUE; +} + +static void +find_remotes_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepo *self; + g_autoptr(GTask) task = NULL; + GCancellable *cancellable; + const FindRemotesData *data; + const OstreeCollectionRef * const *refs; + OstreeAsyncProgress *progress; + g_autoptr(GError) error = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + gsize i; + GHashTableIter iter; + CommitMetadata *commit_metadata; + g_autoptr(PointerTable) refs_and_remotes_table = NULL; /* (element-type commit-checksum) */ + g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */ + g_autoptr(OstreeFetcher) fetcher = NULL; + g_autofree const gchar **ref_to_latest_commit = NULL; /* indexed as @refs; (element-type commit-checksum) */ + gsize n_refs; + const gchar *checksum; + g_autoptr(GPtrArray) remotes_to_remove = NULL; /* (element-type OstreeRemote) */ + g_autoptr(GPtrArray) final_results = NULL; /* (element-type OstreeRepoFinderResult) */ + + task = G_TASK (user_data); + self = OSTREE_REPO (g_task_get_source_object (task)); + cancellable = g_task_get_cancellable (task); + data = g_task_get_task_data (task); + + refs = (const OstreeCollectionRef * const *) data->refs; + progress = data->progress; + + /* Finish finding the remotes. */ + results = ostree_repo_finder_resolve_all_finish (result, &error); + + if (results == NULL) + { + g_task_return_error (task, g_steal_pointer (&error)); + return; + } + + if (results->len == 0) + { + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); + return; + } + + /* Throughout this function, we eliminate invalid results from @results by + * clearing them to %NULL. We cannot remove them from the array, as that messes + * up iteration and stored array indices. Accordingly, we need the free function + * to be %NULL-safe. */ + g_ptr_array_set_free_func (results, (GDestroyNotify) repo_finder_result_free0); + + /* FIXME: Add support for options: + * - override-commit-ids (allow downgrades) + * + * Use case: multiple pulls of separate subdirs; want them to use the same + * configuration. + * Use case: downgrading a flatpak app. + */ + + /* FIXME: In future, we also want to pull static delta superblocks in this + * phase, so that we have all the metadata we need for accurate size + * estimation for the actual pull operation. This should check the + * disable-static-deltas option first. */ + + /* FIXME: We currently do nothing with @progress. */ + + /* Each key must be a pointer to the #CommitMetadata.checksum field of its value. */ + commit_metadatas = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, (GDestroyNotify) commit_metadata_free); + + /* X dimension is an index into @refs. Y dimension is an index into @results. + * Each cell stores the commit checksum which that ref resolves to on that + * remote, or %NULL if the remote doesn’t have that ref. */ + n_refs = g_strv_length ((gchar **) refs); /* it’s not a GStrv, but this works */ + refs_and_remotes_table = pointer_table_new (n_refs, results->len); + remotes_to_remove = g_ptr_array_new_with_free_func (NULL); + + /* Fetch and validate the summary file for each result. */ + /* FIXME: All these downloads could be parallelised; that requires the + * ostree_repo_remote_fetch_summary_with_options() API to be async. */ + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GBytes) summary_bytes = NULL, summary_sig_bytes = NULL; + g_autoptr(GVariant) summary_v = NULL; + guint64 summary_last_modified; + g_autoptr(GVariant) summary_refs = NULL; + g_autoptr(GVariant) additional_metadata_v = NULL; + g_autofree gchar *summary_collection_id = NULL; + g_autoptr(GVariantIter) summary_collection_map = NULL; + gboolean invalid_result = FALSE; + + /* Add the remote to our internal list of remotes, so other libostree + * API can access it. */ + if (!_ostree_repo_add_remote (self, result->remote)) + g_ptr_array_add (remotes_to_remove, result->remote); + + g_debug ("%s: Fetching summary for remote ‘%s’ with keyring ‘%s’.", + G_STRFUNC, result->remote->name, result->remote->keyring); + + /* Download the summary and signature, and validate the signature. This + * will load from the cache if possible. */ + ostree_repo_remote_fetch_summary_with_options (self, + result->remote->name, + NULL, /* no options */ + &summary_bytes, + &summary_sig_bytes, + cancellable, + &error); + + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + goto error; + else if (error != NULL) + { + g_debug ("%s: Failed to download summary for result ‘%s’. Ignoring. %s", + G_STRFUNC, result->remote->name, error->message); + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + g_clear_error (&error); + continue; + } + + /* Check the metadata in the summary file, especially whether it contains + * all the @refs we are interested in. */ + summary_v = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, + summary_bytes, FALSE); + + /* Check the summary’s additional metadata and set up @commit_metadata + * and @refs_and_remotes_table with all the refs listed in the summary + * file which intersect with @refs. */ + additional_metadata_v = g_variant_get_child_value (summary_v, 1); + + if (g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_ID, "s", &summary_collection_id)) + { + summary_refs = g_variant_get_child_value (summary_v, 0); + + if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs, + commit_metadatas, refs_and_remotes_table)) + { + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + continue; + } + } + + if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_COLLECTION_MAP, "a{sa(s(taya{sv}))}", &summary_collection_map)) + summary_collection_map = NULL; + + while (summary_collection_map != NULL && + g_variant_iter_loop (summary_collection_map, "{s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs)) + { + if (!find_remotes_process_refs (self, refs, result, i, summary_collection_id, summary_refs, + commit_metadatas, refs_and_remotes_table)) + { + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + invalid_result = TRUE; + break; + } + } + + if (invalid_result) + continue; + + /* Check the summary timestamp. */ + if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_LAST_MODIFIED, "t", &summary_last_modified)) + summary_last_modified = 0; + else + summary_last_modified = GUINT64_FROM_BE (summary_last_modified); + + /* Update the stored result data. Clear the @ref_to_checksum map, since + * it’s been moved to @refs_and_remotes_table and is now potentially out + * of date. */ + g_clear_pointer (&result->ref_to_checksum, g_hash_table_unref); + result->summary_last_modified = summary_last_modified; + } + + /* Fill in any gaps in the metadata for the most recent commits by pulling + * the commit metadata from the remotes. The ‘most recent commits’ are the + * set of head commits pointed to by the refs we just resolved from the + * summary files. */ + g_hash_table_iter_init (&iter, commit_metadatas); + + while (g_hash_table_iter_next (&iter, (gpointer *) &checksum, (gpointer *) &commit_metadata)) + { + char buf[_OSTREE_LOOSE_PATH_MAX]; + g_autofree gchar *commit_filename = NULL; + g_autoptr(GPtrArray) mirrorlist = NULL; /* (element-type OstreeFetcherURI) */ + g_autoptr(GBytes) commit_bytes = NULL; + g_autoptr(GVariant) commit_v = NULL; + guint64 commit_timestamp; + g_autoptr(GDateTime) dt = NULL; + + /* Already complete? */ + if (commit_metadata->timestamp != 0) + continue; + + _ostree_loose_path (buf, commit_metadata->checksum, OSTREE_OBJECT_TYPE_COMMIT, OSTREE_REPO_MODE_ARCHIVE_Z2); + commit_filename = g_build_filename ("objects", buf, NULL); + + /* For each of the remotes whose summary files contain this ref, try + * downloading the commit metadata until we succeed. Since the results are + * in priority order, the most important remotes are tried first. */ + for (i = 0; i < commit_metadata->refs->len; i++) + { + gsize ref_index = g_array_index (commit_metadata->refs, gsize, i); + gsize j; + + for (j = 0; j < results->len; j++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, j); + + /* Previous error processing this result? */ + if (result == NULL) + continue; + + if (pointer_table_get (refs_and_remotes_table, ref_index, j) != commit_metadata->checksum) + continue; + + g_autofree gchar *uri = NULL; + g_autoptr(OstreeFetcherURI) fetcher_uri = NULL; + + if (!ostree_repo_remote_get_url (self, result->remote->name, + &uri, &error)) + goto error; + + fetcher_uri = _ostree_fetcher_uri_parse (uri, &error); + if (fetcher_uri == NULL) + goto error; + + fetcher = _ostree_repo_remote_new_fetcher (self, result->remote->name, + TRUE, &error); + if (fetcher == NULL) + goto error; + + g_debug ("%s: Fetching metadata for commit ‘%s’ from remote ‘%s’.", + G_STRFUNC, commit_metadata->checksum, result->remote->name); + + /* FIXME: Support remotes which have contenturl, mirrorlist, etc. */ + mirrorlist = g_ptr_array_new_with_free_func ((GDestroyNotify) _ostree_fetcher_uri_free); + g_ptr_array_add (mirrorlist, g_steal_pointer (&fetcher_uri)); + + if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, + mirrorlist, + commit_filename, + FALSE, /* don’t add trailing nul */ + TRUE, /* return NULL on ENOENT */ + &commit_bytes, + 0, /* no maximum size */ + cancellable, + &error)) + goto error; + + glnx_unref_object OstreeGpgVerifyResult *verify_result = NULL; + + verify_result = ostree_repo_verify_commit_for_remote (self, + commit_metadata->checksum, + result->remote->name, + cancellable, + &error); + if (verify_result == NULL) + { + g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum); + goto error; + } + + if (!ostree_gpg_verify_result_require_valid_signature (verify_result, &error)) + { + g_prefix_error (&error, "Commit %s: ", commit_metadata->checksum); + goto error; + } + + if (commit_bytes != NULL) + break; + } + + if (commit_bytes != NULL) + break; + } + + if (commit_bytes == NULL) + { + g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Metadata not found for commit ‘%s’", commit_metadata->checksum); + goto error; + } + + /* Parse the commit metadata. */ + commit_v = g_variant_new_from_bytes (OSTREE_COMMIT_GVARIANT_FORMAT, + commit_bytes, FALSE); + g_variant_get_child (commit_v, 5, "t", &commit_timestamp); + commit_timestamp = GUINT64_FROM_BE (commit_timestamp); + dt = g_date_time_new_from_unix_utc (commit_timestamp); + + if (dt == NULL) + { + g_debug ("%s: Commit ‘%s’ metadata contained timestamp %" G_GUINT64_FORMAT " which is too far in the future. Resetting to 0.", + G_STRFUNC, commit_metadata->checksum, commit_timestamp); + commit_timestamp = 0; + } + + /* Update the #CommitMetadata. */ + commit_metadata->timestamp = commit_timestamp; + } + + /* Find the latest commit for each ref. This is where we resolve the + * differences between remotes: two remotes could both contain ref R, but one + * remote could be outdated compared to the other, and point to an older + * commit. For each ref, we want to find the most recent commit any remote + * points to for it. + * + * @ref_to_latest_commit is indexed by @ref_index, and its values are the + * latest checksum for each ref. */ + ref_to_latest_commit = g_new0 (const gchar *, n_refs); + + for (i = 0; i < n_refs; i++) + { + gsize j; + const gchar *latest_checksum = NULL; + const CommitMetadata *latest_commit_metadata = NULL; + g_autofree gchar *latest_commit_timestamp_str = NULL; + + for (j = 0; j < results->len; j++) + { + const CommitMetadata *candidate_commit_metadata; + const gchar *candidate_checksum; + + candidate_checksum = pointer_table_get (refs_and_remotes_table, i, j); + + if (candidate_checksum == NULL) + continue; + + candidate_commit_metadata = g_hash_table_lookup (commit_metadatas, candidate_checksum); + g_assert (candidate_commit_metadata != NULL); + + if (latest_commit_metadata == NULL || + candidate_commit_metadata->timestamp > latest_commit_metadata->timestamp) + { + latest_checksum = candidate_checksum; + latest_commit_metadata = candidate_commit_metadata; + } + } + + /* @latest_checksum could be %NULL here if there was an error downloading + * the summary or commit metadata files above. */ + ref_to_latest_commit[i] = latest_checksum; + + if (latest_commit_metadata != NULL) + { + latest_commit_timestamp_str = uint64_secs_to_iso8601 (latest_commit_metadata->timestamp); + g_debug ("%s: Latest commit for ref (%s, %s) across all remotes is ‘%s’ with timestamp %s.", + G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name, + latest_checksum, latest_commit_timestamp_str); + } + else + { + g_debug ("%s: Latest commit for ref (%s, %s) is unknown due to failure to download metadata.", + G_STRFUNC, refs[i]->collection_id, refs[i]->ref_name); + } + } + + /* Recombine @commit_metadatas and @results so that each + * #OstreeRepoFinderResult.refs lists the refs for which that remote has the + * latest commits (i.e. it’s not out of date compared to some other remote). */ + final_results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GHashTable) validated_ref_to_checksum = NULL; /* (element-type utf8 utf8) */ + gsize j; + + /* Previous error processing this result? */ + if (result == NULL) + continue; + + /* Map of refs to checksums provided by this result. The checksums should + * be %NULL for each ref unless this result provides the latest checksum. */ + validated_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, + (GDestroyNotify) ostree_collection_ref_free, + g_free); + + for (j = 0; refs[j] != NULL; j++) + { + const gchar *latest_commit_for_ref = ref_to_latest_commit[j]; + + if (pointer_table_get (refs_and_remotes_table, j, i) != latest_commit_for_ref) + latest_commit_for_ref = NULL; + + g_hash_table_insert (validated_ref_to_checksum, ostree_collection_ref_dup (refs[j]), g_strdup (latest_commit_for_ref)); + } + + if (g_hash_table_size (validated_ref_to_checksum) == 0) + { + g_debug ("%s: Omitting remote ‘%s’ from results as none of its refs are new enough.", + G_STRFUNC, result->remote->name); + ostree_repo_finder_result_free (g_steal_pointer (&g_ptr_array_index (results, i))); + continue; + } + + result->ref_to_checksum = g_steal_pointer (&validated_ref_to_checksum); + g_ptr_array_add (final_results, g_steal_pointer (&g_ptr_array_index (results, i))); + } + + /* Ensure the updated results are still in priority order. */ + g_ptr_array_sort (final_results, sort_results_cb); + + /* Remove the remotes we temporarily added. + * FIXME: It would be so much better if we could pass #OstreeRemote pointers + * around internally, to avoid serialising on the global table of them. */ + for (i = 0; i < remotes_to_remove->len; i++) + { + OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i); + _ostree_repo_remove_remote (self, remote); + } + + g_task_return_pointer (task, g_steal_pointer (&final_results), (GDestroyNotify) g_ptr_array_unref); + + return; + +error: + /* Remove the remotes we temporarily added. */ + for (i = 0; i < remotes_to_remove->len; i++) + { + OstreeRemote *remote = g_ptr_array_index (remotes_to_remove, i); + _ostree_repo_remove_remote (self, remote); + } + + g_task_return_error (task, g_steal_pointer (&error)); +} + +/** + * ostree_repo_find_remotes_finish: + * @self: an #OstreeRepo + * @result: the asynchronous result + * @error: return location for a #GError, or %NULL + * + * Finish an asynchronous pull operation started with + * ostree_repo_find_remotes_async(). + * + * Returns: (transfer full) (array zero-terminated=1): a potentially empty array + * of #OstreeRepoFinderResults, followed by a %NULL terminator element; or + * %NULL on error + * Since: 2017.8 + */ +OstreeRepoFinderResult ** +ostree_repo_find_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + g_autoptr(GPtrArray) results = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_find_remotes_async), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + results = g_task_propagate_pointer (G_TASK (result), error); + + if (results != NULL) + { + g_ptr_array_add (results, NULL); /* NULL terminator */ + return (OstreeRepoFinderResult **) g_ptr_array_free (g_steal_pointer (&results), FALSE); + } + else + return NULL; +} + +static void +copy_option (GVariantDict *master_options, + GVariantDict *slave_options, + const gchar *key, + const GVariantType *expected_type) +{ + g_autoptr(GVariant) option_v = g_variant_dict_lookup_value (master_options, key, expected_type); + if (option_v != NULL) + g_variant_dict_insert_value (slave_options, key, g_steal_pointer (&option_v)); +} + +/** + * ostree_repo_pull_from_remotes_async: + * @self: an #OstreeRepo + * @results: (array zero-terminated=1): %NULL-terminated array of remotes to + * pull from, including the refs to pull from each + * @options: (nullable): A GVariant `a{sv}` with an extensible set of flags + * @progress: (nullable): an #OstreeAsyncProgress to update with the operation’s + * progress, or %NULL + * @cancellable: (nullable): a #GCancellable, or %NULL + * @callback: asynchronous completion callback + * @user_data: data to pass to @callback + * + * Pull refs from multiple remotes which have been found using + * ostree_repo_find_remotes_async(). + * + * @results are expected to be in priority order, with the best remotes to pull + * from listed first. ostree_repo_pull_from_remotes_async() will generally pull + * from the remotes in order, but may parallelise its downloads. + * + * If an error is encountered when pulling from a given remote, that remote will + * be ignored and another will be tried instead. If any refs have not been + * downloaded successfully after all remotes have been tried, %G_IO_ERROR_FAILED + * will be returned. The results of any successful downloads will remain cached + * in the local repository. + * + * If @cancellable is cancelled, %G_IO_ERROR_CANCELLED will be returned + * immediately. The results of any successfully completed downloads at that + * point will remain cached in the local repository. + * + * GPG verification of the summary and all commits will be used unconditionally. + * + * The following @options are currently defined: + * + * * `flags` (`i`): #OstreeRepoPullFlags to apply to the pull operation + * * `inherit-transaction` (`b`): %TRUE to inherit an ongoing transaction on + * the #OstreeRepo, rather than encapsulating the pull in a new one + * + * Since: 2017.8 + */ +void +ostree_repo_pull_from_remotes_async (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_return_if_fail (OSTREE_IS_REPO (self)); + g_return_if_fail (results != NULL && results[0] != NULL); + g_return_if_fail (options == NULL || g_variant_is_of_type (options, G_VARIANT_TYPE ("a{sv}"))); + g_return_if_fail (progress == NULL || OSTREE_IS_ASYNC_PROGRESS (progress)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + g_autoptr(GTask) task = NULL; + g_autoptr(GHashTable) refs_pulled = NULL; /* (element-type OstreeCollectionRef gboolean) */ + gsize i, j; + g_autoptr(GString) refs_unpulled_string = NULL; + GHashTableIter iter; + const OstreeCollectionRef *ref; + gpointer is_pulled_pointer; + g_autoptr(GError) local_error = NULL; + g_auto(GVariantDict) options_dict = OT_VARIANT_BUILDER_INITIALIZER; + OstreeRepoPullFlags flags; + gboolean inherit_transaction; + + /* Set up a task for the whole operation. */ + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_pull_from_remotes_async); + + /* Keep track of the set of refs we’ve pulled already. Value is %TRUE if the + * ref has been pulled; %FALSE if it has not. */ + refs_pulled = g_hash_table_new_full (ostree_collection_ref_hash, + ostree_collection_ref_equal, NULL, NULL); + + g_variant_dict_init (&options_dict, options); + + if (!g_variant_dict_lookup (&options_dict, "flags", "i", &flags)) + flags = OSTREE_REPO_PULL_FLAGS_NONE; + if (!g_variant_dict_lookup (&options_dict, "inherit-transaction", "b", &inherit_transaction)) + inherit_transaction = FALSE; + + /* Run all the local pull operations in a single overall transaction. */ + if (!inherit_transaction && + !ostree_repo_prepare_transaction (self, NULL, cancellable, &local_error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + /* FIXME: Rework this code to pull in parallel where possible. At the moment + * we expect the (i == 0) iteration will do all the work (all the refs) and + * subsequent iterations are only there in case of error. + * + * The code is currently all synchronous, too. Making it asynchronous requires + * the underlying pull code to be asynchronous. */ + for (i = 0; results[i] != NULL; i++) + { + const OstreeRepoFinderResult *result = results[i]; + + g_autoptr(GString) refs_to_pull_str = NULL; + g_autoptr(GPtrArray) refs_to_pull = NULL; /* (element-type OstreeCollectionRef) */ + g_auto(GVariantBuilder) refs_to_pull_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_auto(GVariantDict) local_options_dict = OT_VARIANT_BUILDER_INITIALIZER; + g_autoptr(GVariant) local_options = NULL; + const gchar *checksum; + gboolean remove_remote; + + refs_to_pull = g_ptr_array_new_with_free_func (NULL); + refs_to_pull_str = g_string_new (""); + g_variant_builder_init (&refs_to_pull_builder, G_VARIANT_TYPE ("a(sss)")); + + g_hash_table_iter_init (&iter, result->ref_to_checksum); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &checksum)) + { + if (checksum != NULL && + !GPOINTER_TO_INT (g_hash_table_lookup (refs_pulled, ref))) + { + g_ptr_array_add (refs_to_pull, (gpointer) ref); + g_variant_builder_add (&refs_to_pull_builder, "(sss)", + ref->collection_id, ref->ref_name, checksum); + + if (refs_to_pull_str->len > 0) + g_string_append (refs_to_pull_str, ", "); + g_string_append_printf (refs_to_pull_str, "(%s, %s)", + ref->collection_id, ref->ref_name); + } + } + + if (refs_to_pull->len == 0) + { + g_debug ("Ignoring remote ‘%s’ as it has no relevant refs or they " + "have already been pulled.", + result->remote->name); + continue; + } + + /* NULL terminators. */ + g_ptr_array_add (refs_to_pull, NULL); + + g_debug ("Pulling from remote ‘%s’: %s", + result->remote->name, refs_to_pull_str->str); + + /* Set up the pull options. */ + g_variant_dict_init (&local_options_dict, NULL); + + g_variant_dict_insert (&local_options_dict, "flags", "i", OSTREE_REPO_PULL_FLAGS_UNTRUSTED | flags); + g_variant_dict_insert_value (&local_options_dict, "collection-refs", g_variant_builder_end (&refs_to_pull_builder)); + g_variant_dict_insert (&local_options_dict, "gpg-verify", "b", TRUE); + g_variant_dict_insert (&local_options_dict, "gpg-verify-summary", "b", TRUE); + g_variant_dict_insert (&local_options_dict, "inherit-transaction", "b", TRUE); + copy_option (&options_dict, &local_options_dict, "depth", G_VARIANT_TYPE ("i")); + copy_option (&options_dict, &local_options_dict, "disable-static-deltas", G_VARIANT_TYPE ("b")); + copy_option (&options_dict, &local_options_dict, "http-headers", G_VARIANT_TYPE ("a(ss)")); + copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as")); + copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u")); + + local_options = g_variant_dict_end (&local_options_dict); + + /* FIXME: We do nothing useful with @progress at the moment. */ + remove_remote = !_ostree_repo_add_remote (self, result->remote); + ostree_repo_pull_with_options (self, result->remote->name, local_options, + progress, cancellable, &local_error); + if (remove_remote) + _ostree_repo_remove_remote (self, result->remote); + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + if (!inherit_transaction) + ostree_repo_abort_transaction (self, NULL, NULL); + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + for (j = 0; refs_to_pull->pdata[j] != NULL; j++) + g_hash_table_replace (refs_pulled, refs_to_pull->pdata[j], + GINT_TO_POINTER (local_error == NULL)); + + if (local_error != NULL) + { + g_debug ("Failed to pull refs from ‘%s’: %s", + result->remote->name, local_error->message); + g_clear_error (&local_error); + continue; + } + else + { + g_debug ("Pulled refs from ‘%s’.", result->remote->name); + } + } + + /* Commit the transaction. */ + if (!inherit_transaction && + !ostree_repo_commit_transaction (self, NULL, cancellable, &local_error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + return; + } + + /* Any refs left un-downloaded? If so, we’ve failed. */ + g_hash_table_iter_init (&iter, refs_pulled); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, (gpointer *) &is_pulled_pointer)) + { + gboolean is_pulled = GPOINTER_TO_INT (is_pulled_pointer); + + if (is_pulled) + continue; + + if (refs_unpulled_string == NULL) + refs_unpulled_string = g_string_new (""); + else + g_string_append (refs_unpulled_string, ", "); + + g_string_append_printf (refs_unpulled_string, "(%s, %s)", + ref->collection_id, ref->ref_name); + } + + if (refs_unpulled_string != NULL) + { + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to pull some refs from the remotes: %s", + refs_unpulled_string->str); + return; + } + + g_task_return_boolean (task, TRUE); +} + +/** + * ostree_repo_pull_from_remotes_finish: + * @self: an #OstreeRepo + * @result: the asynchronous result + * @error: return location for a #GError, or %NULL + * + * Finish an asynchronous pull operation started with + * ostree_repo_pull_from_remotes_async(). + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.8 + */ +gboolean +ostree_repo_pull_from_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, ostree_repo_pull_from_remotes_async), FALSE); + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + + return g_task_propagate_boolean (G_TASK (result), error); +} + +/* Check whether the given remote exists, has a `collection-id` key set, and it + * equals @collection_id. If so, return %TRUE. Otherwise, %FALSE. */ +static gboolean +check_remote_matches_collection_id (OstreeRepo *repo, + const gchar *remote_name, + const gchar *collection_id) +{ + g_autofree gchar *remote_collection_id = NULL; + + if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL, + &remote_collection_id, NULL) || + remote_collection_id == NULL) + return FALSE; + + return g_str_equal (remote_collection_id, collection_id); +} + +/** + * ostree_repo_resolve_keyring_for_collection: + * @self: an #OstreeRepo + * @collection_id: the collection ID to look up a keyring for + * @cancellable: (nullable): a #GCancellable, or %NULL + * @error: return location for a #GError, or %NULL + * + * Find the GPG keyring for the given @collection_id, using the local + * configuration from the given #OstreeRepo. This will search the configured + * remotes for ones whose `collection-id` key matches @collection_id, and will + * return the GPG keyring from the first matching remote. + * + * If multiple remotes match and have different keyrings, a debug message will + * be emitted, and the first result will be returned. It is expected that the + * keyrings should match. + * + * If no match can be found, a %G_IO_ERROR_NOT_FOUND error will be returned. + * + * Returns: (transfer full): filename of the GPG keyring for @collection_id + * Since: 2017.8 + */ +gchar * +ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, + const gchar *collection_id, + GCancellable *cancellable, + GError **error) +{ + gsize i; + g_auto(GStrv) remotes = NULL; + const OstreeRemote *keyring_remote = NULL; + + g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); + g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL); + g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + /* Look through all the currently configured remotes for the given collection. */ + remotes = ostree_repo_remote_list (self, NULL); + + for (i = 0; remotes != NULL && remotes[i] != NULL; i++) + { + g_autoptr(GError) local_error = NULL; + + if (!check_remote_matches_collection_id (self, remotes[i], collection_id)) + continue; + + if (keyring_remote == NULL) + { + g_debug ("%s: Found match for collection ‘%s’ in remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i]); + keyring_remote = _ostree_repo_get_remote_inherited (self, remotes[i], &local_error); + + if (keyring_remote == NULL) + { + g_debug ("%s: Error loading remote ‘%s’: %s", + G_STRFUNC, remotes[i], local_error->message); + continue; + } + + if (g_strcmp0 (keyring_remote->keyring, "") == 0 || + g_strcmp0 (keyring_remote->keyring, "/dev/null") == 0) + { + g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.", + G_STRFUNC, remotes[i]); + continue; + } + + /* continue so we can catch duplicates */ + } + else + { + g_debug ("%s: Duplicate keyring for collection ‘%s’ in remote ‘%s’." + "Keyring will be loaded from remote ‘%s’.", + G_STRFUNC, collection_id, remotes[i], + keyring_remote->name); + } + } + + if (keyring_remote != NULL) + return g_strdup (keyring_remote->keyring); + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "No keyring found configured locally for collection ‘%s’", + collection_id); + return NULL; + } +} + +#endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ + /** * ostree_repo_remote_fetch_summary_with_options: * @self: Self diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 388d73c4..ea7a7789 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -27,6 +27,7 @@ #include "ostree-async-progress.h" #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include "ostree-ref.h" +#include "ostree-repo-finder.h" #endif #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" @@ -1080,6 +1081,33 @@ ostree_repo_pull_one_dir (OstreeRepo *self, GCancellable *cancellable, GError **error); + + +#if 0 +FIXME +Called with: remote_name, refs, override-commit-ids +or: URL, refs, override-commit-ids +=> we only need refs; could use the remote_name or URL as additional results + +Summary file is downloaded first, so this would result in multiple downloads of +the summary, but we don’t care because of caching. + +Big problem preventing this from being the overall API: presenting the download +sizes in the gnome-software UI before the user chooses to download. + +_OSTREE_PUBLIC +gboolean ostree_repo_find_remotes_squashed (OstreeRepo *self, + const gchar * const *refs, -> options + GVariant *options, + OstreeRepoFinder **finders, -> options + GMainContext *context, -> nope + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error); +#endif + + + _OSTREE_PUBLIC gboolean ostree_repo_pull_with_options (OstreeRepo *self, const char *remote_name_or_baseurl, @@ -1090,6 +1118,39 @@ gboolean ostree_repo_pull_with_options (OstreeRepo *self, #ifdef OSTREE_ENABLE_EXPERIMENTAL_API +_OSTREE_PUBLIC +void ostree_repo_find_remotes_async (OstreeRepo *self, + const OstreeCollectionRef * const *refs, + GVariant *options, + OstreeRepoFinder **finders, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +OstreeRepoFinderResult **ostree_repo_find_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_pull_from_remotes_async (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +gboolean ostree_repo_pull_from_remotes_finish (OstreeRepo *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +gchar *ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, + const gchar *collection_id, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_list_collection_refs (OstreeRepo *self, const char *match_collection_id, diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index c9c3bb75..935707e3 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -37,6 +37,7 @@ #ifdef OSTREE_ENABLE_EXPERIMENTAL_API #include +#include #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */ #include