diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index a7ea85cd..d1c5fd14 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -78,7 +78,8 @@ typedef struct { char *summary_sig_etag; guint64 summary_sig_last_modified; /* seconds since the epoch */ GVariant *summary; - GHashTable *summary_deltas_checksums; + GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */ + gboolean summary_has_deltas; /* True if the summary existed and had a delta index */ GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ GHashTable *verified_commits; /* Set of commits that have been verified */ GHashTable *signapi_verified_commits; /* Map of commits that have been signapi verified */ @@ -93,6 +94,7 @@ typedef struct { GHashTable *requested_fallback_content; /* Maps checksum to itself */ GHashTable *pending_fetch_metadata; /* Map */ GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_delta_indexes; /* Set */ GHashTable *pending_fetch_delta_superblocks; /* Set */ GHashTable *pending_fetch_deltaparts; /* Set */ guint n_outstanding_metadata_fetches; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index bb143136..70cbeca2 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -104,6 +104,14 @@ typedef struct { guint n_retries_remaining; } FetchDeltaSuperData; +typedef struct { + OtPullData *pull_data; + char *from_revision; + char *to_revision; + OstreeCollectionRef *requested_ref; /* (nullable) */ + guint n_retries_remaining; +} FetchDeltaIndexData; + static void variant_or_null_unref (gpointer data) { @@ -116,6 +124,8 @@ static void start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch); static void start_fetch_delta_superblock (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); +static void start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static gboolean fetcher_queue_is_full (OtPullData *pull_data); static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, @@ -135,6 +145,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da static void enqueue_one_object_request_s (OtPullData *pull_data, FetchObjectData *fetch_data); +static void enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data, @@ -149,6 +161,11 @@ static gboolean scan_one_metadata_object (OtPullData *pull_data, GCancellable *cancellable, GError **error); static void scan_object_queue_data_free (ScanObjectQueueData *scan_data); +static gboolean initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error); static gboolean update_progress (gpointer user_data) @@ -287,6 +304,7 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); g_queue_clear (&pull_data->scan_object_queue); g_hash_table_remove_all (pull_data->pending_fetch_metadata); + g_hash_table_remove_all (pull_data->pending_fetch_delta_indexes); g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks); g_hash_table_remove_all (pull_data->pending_fetch_deltaparts); g_hash_table_remove_all (pull_data->pending_fetch_content); @@ -320,6 +338,16 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_variant_unref (objname); } + /* Next, process delta index requests */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_indexes); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + FetchDeltaIndexData *fetch = key; + g_hash_table_iter_steal (&hiter); + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch)); + } + /* Next, process delta superblock requests */ g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks); while (!fetcher_queue_is_full (pull_data) && @@ -2469,6 +2497,16 @@ fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data) g_free (fetch_data); } +static void +fetch_delta_index_data_free (FetchDeltaIndexData *fetch_data) +{ + g_free (fetch_data->from_revision); + g_free (fetch_data->to_revision); + if (fetch_data->requested_ref) + ostree_collection_ref_free (fetch_data->requested_ref); + g_free (fetch_data); +} + static void set_required_deltas_error (GError **error, const char *from_revision, @@ -2629,6 +2667,146 @@ validate_variant_is_csum (GVariant *csum, return ostree_validate_structureof_csum_v (csum, error); } +static gboolean +collect_available_deltas_for_pull (OtPullData *pull_data, + GVariant *deltas, + GError **error) +{ + gsize n; + + n = deltas ? g_variant_n_children (deltas) : 0; + for (gsize i = 0; i < n; i++) + { + const char *delta; + g_autoptr(GVariant) csum_v = NULL; + g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); + + g_variant_get_child (ref, 0, "&s", &delta); + g_variant_get_child (ref, 1, "v", &csum_v); + + if (!validate_variant_is_csum (csum_v, error)) + return FALSE; + + guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); + memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); + g_hash_table_insert (pull_data->summary_deltas_checksums, + g_strdup (delta), + csum_data); + } + + return TRUE; +} + +static void +on_delta_index_fetched (GObject *src, + GAsyncResult *res, + gpointer data) + +{ + FetchDeltaIndexData *fetch_data = data; + OtPullData *pull_data = fetch_data->pull_data; + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + g_autoptr(GBytes) delta_index_data = NULL; + const char *from_revision = fetch_data->from_revision; + const char *to_revision = fetch_data->to_revision; + + if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, + res, + &delta_index_data, + NULL, NULL, NULL, + error)) + { + if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + goto out; + g_clear_error (&local_error); + + /* below call to initiate_delta_request() will fail finding the delta and fall back to commit */ + } + else + { + g_autoptr(GVariant) delta_index = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, delta_index_data, FALSE)); + g_autoptr(GVariant) deltas = g_variant_lookup_value (delta_index, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); + + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; + } + + if (!initiate_delta_request (pull_data, + fetch_data->requested_ref, + to_revision, + from_revision, + &local_error)) + goto out; + + out: + g_assert (pull_data->n_outstanding_metadata_fetches > 0); + pull_data->n_outstanding_metadata_fetches--; + + if (local_error == NULL) + pull_data->n_fetched_metadata++; + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + + g_clear_pointer (&fetch_data, fetch_delta_index_data_free); +} + +static void +start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + g_autofree char *delta_name = + _ostree_get_relative_static_delta_index_path (fetch_data->to_revision); + _ostree_fetcher_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, + OSTREE_MAX_METADATA_SIZE, + 0, pull_data->cancellable, + on_delta_index_fetched, + g_steal_pointer (&fetch_data)); + pull_data->n_outstanding_metadata_fetches++; + pull_data->n_requested_metadata++; +} + +static void +enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of static delta index to %s", + fetch_data->to_revision); + + g_hash_table_add (pull_data->pending_fetch_delta_indexes, + g_steal_pointer (&fetch_data)); + } + else + { + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch_data)); + } +} + +/* Start a request for a static delta index */ +static void +enqueue_one_static_delta_index_request (OtPullData *pull_data, + const char *to_revision, + const char *from_revision, + const OstreeCollectionRef *ref) +{ + FetchDeltaIndexData *fdata = g_new0(FetchDeltaIndexData, 1); + fdata->pull_data = pull_data; + fdata->from_revision = g_strdup (from_revision); + fdata->to_revision = g_strdup (to_revision); + fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fdata->n_retries_remaining = pull_data->n_network_retries; + + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fdata)); +} + static gboolean _ostree_repo_verify_summary (OstreeRepo *self, const char *name, @@ -3259,6 +3437,65 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, return TRUE; } +static gboolean +initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error) +{ + DeltaSearchResult deltares; + + /* Look for a delta to @to_revision in the summary data */ + if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, + pull_data->cancellable, error)) + return FALSE; + + switch (deltares.result) + { + case DELTA_SEARCH_RESULT_NO_MATCH: + { + if (pull_data->require_static_deltas) /* No deltas found; are they required? */ + { + set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); + return FALSE; + } + else /* No deltas, fall back to object fetches. */ + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + break; + case DELTA_SEARCH_RESULT_FROM: + enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); + break; + case DELTA_SEARCH_RESULT_SCRATCH: + { + /* If a from-scratch delta is available, we don’t want to use it if + * the ref already exists locally, since we are likely only a few + * commits out of date; so doing an object pull is likely more + * bandwidth efficient. */ + if (delta_from_revision != NULL) + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + else + enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); + } + break; + case DELTA_SEARCH_RESULT_UNCHANGED: + { + /* If we already have the commit, here things get a little special; we've historically + * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas + * path, we don't, under the assumption the user wants as little network traffic as + * possible. + */ + if (pull_data->require_static_deltas) + break; + else + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + } + + return TRUE; +} + /* * initiate_request: * @ref: Optional ref name and collection ID @@ -3301,53 +3538,14 @@ initiate_request (OtPullData *pull_data, /* If we have a summary, we can use the newer logic */ if (pull_data->summary) { - DeltaSearchResult deltares; - - /* Look for a delta to @to_revision in the summary data */ - if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, - pull_data->cancellable, error)) - return FALSE; - - switch (deltares.result) + if (!pull_data->summary_has_deltas) { - case DELTA_SEARCH_RESULT_NO_MATCH: - { - if (pull_data->require_static_deltas) /* No deltas found; are they required? */ - { - set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); - return FALSE; - } - else /* No deltas, fall back to object fetches. */ - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } - break; - case DELTA_SEARCH_RESULT_FROM: - enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); - break; - case DELTA_SEARCH_RESULT_SCRATCH: - { - /* If a from-scratch delta is available, we don’t want to use it if - * the ref already exists locally, since we are likely only a few - * commits out of date; so doing an object pull is likely more - * bandwidth efficient. */ - if (delta_from_revision != NULL) - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - else - enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); - } - break; - case DELTA_SEARCH_RESULT_UNCHANGED: - { - /* If we already have the commit, here things get a little special; we've historically - * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas - * path, we don't, under the assumption the user wants as little network traffic as - * possible. - */ - if (pull_data->require_static_deltas) - break; - else - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } + enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref); + } + else + { + if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error)) + return FALSE; } } else if (ref != NULL) @@ -3657,6 +3855,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_delta_indexes = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_index_data_free, NULL); pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); @@ -4314,25 +4513,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, } deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); - n = deltas ? g_variant_n_children (deltas) : 0; - for (i = 0; i < n; i++) - { - const char *delta; - g_autoptr(GVariant) csum_v = NULL; - g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); - - g_variant_get_child (ref, 0, "&s", &delta); - g_variant_get_child (ref, 1, "v", &csum_v); - - if (!validate_variant_is_csum (csum_v, error)) - goto out; - - guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); - memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); - g_hash_table_insert (pull_data->summary_deltas_checksums, - g_strdup (delta), - csum_data); - } + pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0; + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; } if (pull_data->summary && @@ -4900,6 +5083,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_delta_indexes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref); g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);