Merge pull request #2188 from alexlarsson/delta-indexes

Add indexes for deltas outside of the summary

Makefile-libostree.am

@@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \
 endif # USE_GPGME
 
 symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
-#if BUILDOPT_IS_DEVEL_BUILD
+if BUILDOPT_IS_DEVEL_BUILD
-#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
-#endif
+endif
 # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
 wl_versionscript_arg = -Wl,--version-script=
 EXTRA_DIST += \

apidoc/ostree-sections.txt

@@ -412,6 +412,8 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE
 ostree_repo_list_objects
 ostree_repo_list_commit_objects_starting_with
 ostree_repo_list_static_delta_names
+ostree_repo_list_static_delta_indexes
+ostree_repo_static_delta_reindex
 OstreeStaticDeltaGenerateOpt
 ostree_repo_static_delta_generate
 ostree_repo_static_delta_execute_offline_with_signature
@@ -445,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed
 ostree_repo_sign_commit
 ostree_repo_append_gpg_signature
 ostree_repo_add_gpg_signature_summary
+ostree_repo_gpg_sign_data
 ostree_repo_gpg_verify_data
 ostree_repo_verify_commit
 ostree_repo_verify_commit_ext

man/ostree.repo-config.xml

@@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA.
         costly).
         </para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>no-deltas-in-summary</varname></term>
+        <listitem><para>Boolean value controlling whether OSTree should skip
+        putting an index of available deltas in the summary file. Defaults to false.
+        </para>
+        <para>
+        Since 2020.7 OSTree can use delta indexes outside the summary file,
+        making the summary file smaller (especially for larger repositories). However
+        by default we still create the index in the summary file to make older clients
+        work. If you know all clients will be 2020.7 later you can enable this to
+        save network bandwidth.
+        </para></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

src/libostree/libostree-devel.sym

@@ -17,9 +17,12 @@
   Boston, MA 02111-1307, USA.
 ***/
 
-/* Copy the bits below and uncomment the include in Makefile-libostree.am
+LIBOSTREE_2020.8 {
-   when adding a symbol.
+global:
-*/
+  ostree_repo_list_static_delta_indexes;
+  ostree_repo_static_delta_reindex;
+  ostree_repo_gpg_sign_data;
+} LIBOSTREE_2020.7;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste

src/libostree/ostree-core-private.h

@@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
                                              const char        *to,
                                              guint              i);
 
+char *
+_ostree_get_relative_static_delta_index_path (const char        *to);
+
 static inline char * _ostree_get_commitpartial_path (const char *checksum)
 {
   return g_strconcat ("state/", checksum, ".commitpartial", NULL);

src/libostree/ostree-core.c

@@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char         *checksum,
   return g_string_free (path, FALSE);
 }
 
-char *
+static GString *
-_ostree_get_relative_static_delta_path (const char *from,
+static_delta_path_base (const char *dir,
-                                        const char *to,
+                        const char *from,
-                                        const char *target)
+                        const char *to)
 {
   guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
   char to_b64[44];
   guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
-  GString *ret = g_string_new ("deltas/");
+  GString *ret = g_string_new (dir);
 
   ostree_checksum_inplace_to_bytes (to, csum_to);
   ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
@@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from,
     g_string_append_c (ret, '/');
   g_string_append (ret, to_b64 + 2);
 
+  return ret;
+}
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+                                        const char *to,
+                                        const char *target)
+{
+  GString *ret = static_delta_path_base ("deltas/", from, to);
+
   if (target != NULL)
     {
       g_string_append_c (ret, '/');
@@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
   return _ostree_get_relative_static_delta_path (from, to, partstr);
 }
 
+char *
+_ostree_get_relative_static_delta_index_path (const char *to)
+{
+  GString *ret = static_delta_path_base ("delta-indexes/", NULL, to);
+
+  g_string_append (ret, ".index");
+
+  return g_string_free (ret, FALSE);
+}
+
 gboolean
 _ostree_parse_delta_name (const char   *delta_name,
                           char        **out_from,

src/libostree/ostree-repo-private.h

@@ -57,6 +57,7 @@ G_BEGIN_DECLS
 #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map"
 #define OSTREE_SUMMARY_MODE "ostree.summary.mode"
 #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits"
+#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas"
 
 #define _OSTREE_PAYLOAD_LINK_PREFIX "../"
 #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1)

src/libostree/ostree-repo-pull-private.h

@@ -78,7 +78,9 @@ typedef struct {
   char             *summary_sig_etag;
   guint64           summary_sig_last_modified;  /* seconds since the epoch */
   GVariant         *summary;
-  GHashTable       *summary_deltas_checksums;
+  GHashTable       *summary_deltas_checksums; /* Filled from summary and delta indexes */
+  gboolean          summary_has_deltas; /* True if the summary existed and had a delta index */
+  gboolean          has_indexed_deltas;
   GHashTable       *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
   GHashTable       *verified_commits; /* Set<checksum> of commits that have been verified */
   GHashTable       *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */
@@ -93,6 +95,7 @@ typedef struct {
   GHashTable       *requested_fallback_content; /* Maps checksum to itself */
   GHashTable       *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
   GHashTable       *pending_fetch_content; /* Map<checksum,FetchObjectData> */
+  GHashTable       *pending_fetch_delta_indexes; /* Set<FetchDeltaIndexData> */
   GHashTable       *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
   GHashTable       *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
   guint             n_outstanding_metadata_fetches;

src/libostree/ostree-repo-pull.c

@@ -104,6 +104,14 @@ typedef struct {
   guint n_retries_remaining;
 } FetchDeltaSuperData;
 
+typedef struct {
+  OtPullData *pull_data;
+  char *from_revision;
+  char *to_revision;
+  OstreeCollectionRef *requested_ref;  /* (nullable) */
+  guint n_retries_remaining;
+} FetchDeltaIndexData;
+
 static void
 variant_or_null_unref (gpointer data)
 {
@@ -116,6 +124,8 @@ static void start_fetch_deltapart (OtPullData *pull_data,
                                    FetchStaticDeltaData *fetch);
 static void start_fetch_delta_superblock (OtPullData          *pull_data,
                                           FetchDeltaSuperData *fetch_data);
+static void start_fetch_delta_index (OtPullData          *pull_data,
+                                     FetchDeltaIndexData *fetch_data);
 static gboolean fetcher_queue_is_full (OtPullData *pull_data);
 static void queue_scan_one_metadata_object (OtPullData                *pull_data,
                                             const char                *csum,
@@ -135,6 +145,8 @@ static void queue_scan_one_metadata_object_c (OtPullData                *pull_da
 
 static void enqueue_one_object_request_s (OtPullData      *pull_data,
                                           FetchObjectData *fetch_data);
+static void enqueue_one_static_delta_index_request_s (OtPullData          *pull_data,
+                                                      FetchDeltaIndexData *fetch_data);
 static void enqueue_one_static_delta_superblock_request_s (OtPullData          *pull_data,
                                                            FetchDeltaSuperData *fetch_data);
 static void enqueue_one_static_delta_part_request_s (OtPullData           *pull_data,
@@ -149,6 +161,11 @@ static gboolean scan_one_metadata_object (OtPullData                 *pull_data,
                                           GCancellable               *cancellable,
                                           GError                    **error);
 static void scan_object_queue_data_free (ScanObjectQueueData *scan_data);
+static gboolean initiate_delta_request (OtPullData                *pull_data,
+                                        const OstreeCollectionRef *ref,
+                                        const char                *to_revision,
+                                        const char                 *delta_from_revision,
+                                        GError                    **error);
 
 static gboolean
 update_progress (gpointer user_data)
@@ -287,6 +304,7 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
       g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);
       g_queue_clear (&pull_data->scan_object_queue);
       g_hash_table_remove_all (pull_data->pending_fetch_metadata);
+      g_hash_table_remove_all (pull_data->pending_fetch_delta_indexes);
       g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks);
       g_hash_table_remove_all (pull_data->pending_fetch_deltaparts);
       g_hash_table_remove_all (pull_data->pending_fetch_content);
@@ -320,6 +338,16 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
           g_variant_unref (objname);
         }
 
+      /* Next, process delta index requests */
+      g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_indexes);
+      while (!fetcher_queue_is_full (pull_data) &&
+             g_hash_table_iter_next (&hiter, &key, &value))
+        {
+          FetchDeltaIndexData *fetch = key;
+          g_hash_table_iter_steal (&hiter);
+          start_fetch_delta_index (pull_data, g_steal_pointer (&fetch));
+        }
+
       /* Next, process delta superblock requests */
       g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks);
       while (!fetcher_queue_is_full (pull_data) &&
@@ -2117,6 +2145,7 @@ start_fetch_deltapart (OtPullData *pull_data,
                        FetchStaticDeltaData *fetch)
 {
   g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i);
+  g_debug ("starting fetch of deltapart %s", deltapart_path);
   pull_data->n_outstanding_deltapart_fetches++;
   g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
   _ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
@@ -2469,6 +2498,16 @@ fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data)
   g_free (fetch_data);
 }
 
+static void
+fetch_delta_index_data_free (FetchDeltaIndexData *fetch_data)
+{
+  g_free (fetch_data->from_revision);
+  g_free (fetch_data->to_revision);
+  if (fetch_data->requested_ref)
+    ostree_collection_ref_free (fetch_data->requested_ref);
+  g_free (fetch_data);
+}
+
 static void
 set_required_deltas_error (GError **error,
                            const char *from_revision,
@@ -2570,6 +2609,7 @@ start_fetch_delta_superblock (OtPullData          *pull_data,
   g_autofree char *delta_name =
     _ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision,
                                                        fetch_data->to_revision);
+  g_debug ("starting fetch of delta superblock %s", delta_name);
   _ostree_fetcher_request_to_membuf (pull_data->fetcher,
                                      pull_data->content_mirrorlist,
                                      delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
@@ -2629,6 +2669,147 @@ validate_variant_is_csum (GVariant       *csum,
   return ostree_validate_structureof_csum_v (csum, error);
 }
 
+static gboolean
+collect_available_deltas_for_pull (OtPullData *pull_data,
+                                   GVariant   *deltas,
+                                   GError    **error)
+{
+  gsize n;
+
+  n = deltas ? g_variant_n_children (deltas) : 0;
+  for (gsize i = 0; i < n; i++)
+    {
+      const char *delta;
+      g_autoptr(GVariant) csum_v = NULL;
+      g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
+
+      g_variant_get_child (ref, 0, "&s", &delta);
+      g_variant_get_child (ref, 1, "v", &csum_v);
+
+      if (!validate_variant_is_csum (csum_v, error))
+        return FALSE;
+
+      guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
+      memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32);
+      g_hash_table_insert (pull_data->summary_deltas_checksums,
+                           g_strdup (delta),
+                           csum_data);
+    }
+
+  return TRUE;
+}
+
+static void
+on_delta_index_fetched (GObject   *src,
+                        GAsyncResult *res,
+                        gpointer      data)
+
+{
+  FetchDeltaIndexData *fetch_data = data;
+  OtPullData *pull_data = fetch_data->pull_data;
+  g_autoptr(GError) local_error = NULL;
+  GError **error = &local_error;
+  g_autoptr(GBytes) delta_index_data = NULL;
+  const char *from_revision = fetch_data->from_revision;
+  const char *to_revision = fetch_data->to_revision;
+
+  if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
+                                                 res,
+                                                 &delta_index_data,
+                                                 NULL, NULL, NULL,
+                                                 error))
+    {
+      if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+        goto out;
+      g_clear_error (&local_error);
+
+      /* below call to initiate_delta_request() will fail finding the delta and fall back to commit */
+    }
+  else
+    {
+      g_autoptr(GVariant) delta_index = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, delta_index_data, FALSE));
+      g_autoptr(GVariant) deltas = g_variant_lookup_value (delta_index, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
+
+      if (!collect_available_deltas_for_pull (pull_data, deltas, error))
+        goto out;
+    }
+
+  if (!initiate_delta_request (pull_data,
+                               fetch_data->requested_ref,
+                               to_revision,
+                               from_revision,
+                               &local_error))
+    goto out;
+
+ out:
+  g_assert (pull_data->n_outstanding_metadata_fetches > 0);
+  pull_data->n_outstanding_metadata_fetches--;
+
+  if (local_error == NULL)
+    pull_data->n_fetched_metadata++;
+
+  if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--))
+    enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fetch_data));
+  else
+    check_outstanding_requests_handle_error (pull_data, &local_error);
+
+  g_clear_pointer (&fetch_data, fetch_delta_index_data_free);
+}
+
+static void
+start_fetch_delta_index (OtPullData          *pull_data,
+                         FetchDeltaIndexData *fetch_data)
+{
+  g_autofree char *delta_name =
+    _ostree_get_relative_static_delta_index_path (fetch_data->to_revision);
+  g_debug ("starting fetch of delta index %s", delta_name);
+  _ostree_fetcher_request_to_membuf (pull_data->fetcher,
+                                     pull_data->content_mirrorlist,
+                                     delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
+                                     NULL, 0,
+                                     OSTREE_MAX_METADATA_SIZE,
+                                     0, pull_data->cancellable,
+                                     on_delta_index_fetched,
+                                     g_steal_pointer (&fetch_data));
+  pull_data->n_outstanding_metadata_fetches++;
+  pull_data->n_requested_metadata++;
+}
+
+static void
+enqueue_one_static_delta_index_request_s (OtPullData          *pull_data,
+                                          FetchDeltaIndexData *fetch_data)
+{
+  if (fetcher_queue_is_full (pull_data))
+    {
+      g_debug ("queuing fetch of static delta index to %s",
+               fetch_data->to_revision);
+
+      g_hash_table_add (pull_data->pending_fetch_delta_indexes,
+                        g_steal_pointer (&fetch_data));
+    }
+  else
+    {
+      start_fetch_delta_index (pull_data, g_steal_pointer (&fetch_data));
+    }
+}
+
+/* Start a request for a static delta index */
+static void
+enqueue_one_static_delta_index_request (OtPullData                *pull_data,
+                                        const char                *to_revision,
+                                        const char                *from_revision,
+                                        const OstreeCollectionRef *ref)
+{
+  FetchDeltaIndexData *fdata = g_new0(FetchDeltaIndexData, 1);
+  fdata->pull_data = pull_data;
+  fdata->from_revision = g_strdup (from_revision);
+  fdata->to_revision = g_strdup (to_revision);
+  fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL;
+  fdata->n_retries_remaining = pull_data->n_network_retries;
+
+  enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fdata));
+}
+
 static gboolean
 _ostree_repo_verify_summary (OstreeRepo   *self,
                              const char   *name,
@@ -3259,47 +3440,12 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name,
   return TRUE;
 }
 
-/*
- * initiate_request:
- * @ref: Optional ref name and collection ID
- * @to_revision: Target commit revision we want to fetch
- *
- * Start a request for either a ref or a commit.  In the
- * ref case, we know both the name and the target commit.
- *
- * This function primarily handles the semantics around
- * `disable_static_deltas` and `require_static_deltas`.
- */
 static gboolean
-initiate_request (OtPullData                 *pull_data,
+initiate_delta_request (OtPullData                *pull_data,
                         const OstreeCollectionRef *ref,
                         const char                *to_revision,
+                        const char                 *delta_from_revision,
                         GError                    **error)
-{
-  g_autofree char *delta_from_revision = NULL;
-
-  /* Are deltas disabled?  OK, just start an object fetch and be done */
-  if (pull_data->disable_static_deltas)
-    {
-      queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
-      return TRUE;
-    }
-
-  /* If doing a delta from a ref, look up the from-revision, since we need it
-   * on most paths below. */
-  if (ref != NULL)
-    {
-      g_autofree char *refspec = NULL;
-      if (pull_data->remote_name != NULL)
-        refspec = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name);
-      if (!ostree_repo_resolve_rev (pull_data->repo,
-                                    refspec ?: ref->ref_name, TRUE,
-                                    &delta_from_revision, error))
-        return FALSE;
-    }
-
-  /* If we have a summary, we can use the newer logic */
-  if (pull_data->summary)
 {
   DeltaSearchResult deltares;
 
@@ -3349,6 +3495,60 @@ initiate_request (OtPullData                 *pull_data,
           queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
       }
     }
+
+  return TRUE;
+}
+
+/*
+ * initiate_request:
+ * @ref: Optional ref name and collection ID
+ * @to_revision: Target commit revision we want to fetch
+ *
+ * Start a request for either a ref or a commit.  In the
+ * ref case, we know both the name and the target commit.
+ *
+ * This function primarily handles the semantics around
+ * `disable_static_deltas` and `require_static_deltas`.
+ */
+static gboolean
+initiate_request (OtPullData                 *pull_data,
+                  const OstreeCollectionRef  *ref,
+                  const char                 *to_revision,
+                  GError                    **error)
+{
+  g_autofree char *delta_from_revision = NULL;
+
+  /* Are deltas disabled?  OK, just start an object fetch and be done */
+  if (pull_data->disable_static_deltas)
+    {
+      queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref);
+      return TRUE;
+    }
+
+  /* If doing a delta from a ref, look up the from-revision, since we need it
+   * on most paths below. */
+  if (ref != NULL)
+    {
+      g_autofree char *refspec = NULL;
+      if (pull_data->remote_name != NULL)
+        refspec = g_strdup_printf ("%s:%s", pull_data->remote_name, ref->ref_name);
+      if (!ostree_repo_resolve_rev (pull_data->repo,
+                                    refspec ?: ref->ref_name, TRUE,
+                                    &delta_from_revision, error))
+        return FALSE;
+    }
+
+  /* If we have a summary or delta index, we can use the newer logic.
+   * We prefer the index as it might have more deltas than the summary
+   * (i.e. leave some deltas out of summary to make it smaller). */
+  if (pull_data->has_indexed_deltas)
+    {
+      enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref);
+    }
+  else if (pull_data->summary_has_deltas)
+    {
+      if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error))
+        return FALSE;
     }
   else if (ref != NULL)
     {
@@ -3392,6 +3592,20 @@ initiate_request (OtPullData                 *pull_data,
   return TRUE;
 }
 
+static gboolean
+all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type OstreeCollectionRef utf8) */)
+{
+  GLNX_HASH_TABLE_FOREACH_KV (requested_refs, const OstreeCollectionRef*, ref,
+                              const char*, override_commitid)
+    {
+      /* Note: "" override means whatever is latest */
+      if (override_commitid == NULL || *override_commitid == 0)
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
 /* ------------------------------------------------------------------------------------------
  * Below is the libsoup-invariant API; these should match
  * the stub functions in the #else clause
@@ -3497,9 +3711,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   gboolean opt_ref_keyring_map_set = FALSE;
   gboolean disable_sign_verify = FALSE;
   gboolean disable_sign_verify_summary = FALSE;
+  gboolean need_summary = FALSE;
   const char *main_collection_id = NULL;
   const char *url_override = NULL;
   gboolean inherit_transaction = FALSE;
+  gboolean require_summary_for_mirror = FALSE;
   g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL;  /* (element-type OstreeCollectionRef utf8) */
   gsize i;
   g_autofree char **opt_localcache_repos = NULL;
@@ -3511,6 +3727,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
    */
   const char *the_ref_to_fetch = NULL;
   OstreeRepoTransactionStats tstats = { 0, };
+  gboolean remote_mode_loaded = FALSE;
 
   /* Default */
   pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE;
@@ -3657,6 +3874,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
                                                              (GDestroyNotify)g_variant_unref,
                                                              (GDestroyNotify)fetch_object_data_free);
+  pull_data->pending_fetch_delta_indexes = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_index_data_free, NULL);
   pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL);
   pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
 
@@ -3933,8 +4151,98 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   if (pull_data->is_commit_only)
     pull_data->disable_static_deltas = TRUE;
 
+  /* Compute the set of collection-refs (and optional commit id) to fetch */
+
+  if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
+    {
+      require_summary_for_mirror = TRUE;
+    }
+  else if (opt_collection_refs_set)
+    {
+      const gchar *collection_id, *ref_name, *checksum;
+
+      while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
+        {
+          if (!ostree_validate_rev (ref_name, error))
+            goto out;
+          g_hash_table_insert (requested_refs_to_fetch,
+                               ostree_collection_ref_new (collection_id, ref_name),
+                               (*checksum != '\0') ? g_strdup (checksum) : NULL);
+        }
+    }
+  else if (refs_to_fetch != NULL)
+    {
+      char **strviter = refs_to_fetch;
+      char **commitid_strviter = override_commit_ids ?: NULL;
+
+      while (*strviter)
+        {
+          const char *branch = *strviter;
+
+          if (ostree_validate_checksum_string (branch, NULL))
+            {
+              char *key = g_strdup (branch);
+              g_hash_table_add (commits_to_fetch, key);
+            }
+          else
+            {
+              if (!ostree_validate_rev (branch, error))
+                goto out;
+              char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
+              g_hash_table_insert (requested_refs_to_fetch,
+                                   ostree_collection_ref_new (NULL, branch), commitid);
+            }
+
+          strviter++;
+          if (commitid_strviter)
+            commitid_strviter++;
+        }
+    }
+  else
+    {
+      char **branches_iter;
+
+      branches_iter = configured_branches;
+
+      if (!(branches_iter && *branches_iter))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "No configured branches for remote %s", remote_name_or_baseurl);
+          goto out;
+        }
+      for (;branches_iter && *branches_iter; branches_iter++)
+        {
+          const char *branch = *branches_iter;
+
+          g_hash_table_insert (requested_refs_to_fetch,
+                               ostree_collection_ref_new (NULL, branch), NULL);
+        }
+    }
+
+  /* Deltas are necessary when mirroring or resolving a requested ref to a commit.
+   * We try to avoid loading the potentially large summary if it is not needed. */
+  need_summary = require_summary_for_mirror || !all_requested_refs_have_commit (requested_refs_to_fetch) || summary_sig_bytes_v != NULL;
+
+  /* If we don't have indexed deltas, we need the summary for deltas, so check
+   * the config file for support.
+   * NOTE: Avoid download if we don't need deltas */
+  if (!need_summary && !pull_data->disable_static_deltas)
+    {
+      if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
+        goto out;
+
+      /* Check if remote has delta indexes outside summary */
+      if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "indexed-deltas", FALSE,
+                                                &pull_data->has_indexed_deltas, error))
+        goto out;
+
+      if (!pull_data->has_indexed_deltas)
+        need_summary = TRUE;
+   }
+
   pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
 
+  if (need_summary)
     {
       g_autoptr(GBytes) bytes_sig = NULL;
       gboolean summary_sig_not_modified = FALSE;
@@ -3945,7 +4253,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       g_autoptr(GVariant) deltas = NULL;
       g_autoptr(GVariant) additional_metadata = NULL;
       gboolean summary_from_cache = FALSE;
-    gboolean remote_mode_loaded = FALSE;
       gboolean tombstone_commits = FALSE;
 
       if (summary_sig_bytes_v)
@@ -4038,6 +4345,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
           g_clear_pointer (&summary_etag, g_free);
           summary_last_modified = 0;
+
           if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
                                                            pull_data->meta_mirrorlist,
                                                            "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
@@ -4070,10 +4378,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         }
 #endif /* OSTREE_DISABLE_GPGME */
 
-    if (!bytes_summary && pull_data->require_static_deltas)
+      if (!bytes_summary && require_summary_for_mirror)
         {
-        g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                     "Fetch configured to require static deltas, but no summary found");
+                       "Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
           goto out;
         }
 
@@ -4160,7 +4468,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
               signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT,
                                                      bytes_sig, FALSE);
 
-
               g_assert (pull_data->signapi_summary_verifiers);
               if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error))
                 {
@@ -4314,25 +4621,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
             }
 
           deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}"));
-        n = deltas ? g_variant_n_children (deltas) : 0;
+          pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0;
-        for (i = 0; i < n; i++)
+          if (!collect_available_deltas_for_pull (pull_data, deltas, error))
-          {
-            const char *delta;
-            g_autoptr(GVariant) csum_v = NULL;
-            g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
-
-            g_variant_get_child (ref, 0, "&s", &delta);
-            g_variant_get_child (ref, 1, "v", &csum_v);
-
-            if (!validate_variant_is_csum (csum_v, error))
             goto out;
 
-            guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
+          g_variant_lookup (additional_metadata, OSTREE_SUMMARY_INDEXED_DELTAS, "b", &pull_data->has_indexed_deltas);
-            memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32);
-            g_hash_table_insert (pull_data->summary_deltas_checksums,
-                                 g_strdup (delta),
-                                 csum_data);
-          }
         }
 
       if (pull_data->summary &&
@@ -4344,13 +4637,23 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
           pull_data->has_tombstone_commits = tombstone_commits;
           remote_mode_loaded = TRUE;
         }
-    else if (pull_data->remote_repo_local == NULL)
+    }
+
+  if (pull_data->require_static_deltas && !pull_data->has_indexed_deltas && !pull_data->summary_has_deltas)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
+                   "Fetch configured to require static deltas, but no summary deltas or delta index found");
+      goto out;
+    }
+
+  if (remote_mode_loaded && pull_data->remote_repo_local == NULL)
       {
         /* Fall-back path which loads the necessary config from the remote’s
-         * `config` file. Doing so is deprecated since it means an
+         * `config` file (unless we already read it above). Doing so is deprecated since it means an
          * additional round trip to the remote for each pull. No need to do
          * it for local pulls. */
-        if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
+        if (remote_config == NULL &&
+            !load_remote_repo_config (pull_data, &remote_config, cancellable, error))
           goto out;
 
         if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
@@ -4374,79 +4677,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                    remote_mode_str);
       goto out;
     }
-  }
-
-  if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches)
-    {
-      if (!bytes_summary)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Fetching all refs was requested in mirror mode, but remote repository does not have a summary");
-          goto out;
-        }
-
-    }
-  else if (opt_collection_refs_set)
-    {
-      const gchar *collection_id, *ref_name, *checksum;
-
-      while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
-        {
-          if (!ostree_validate_rev (ref_name, error))
-            goto out;
-          g_hash_table_insert (requested_refs_to_fetch,
-                               ostree_collection_ref_new (collection_id, ref_name),
-                               (*checksum != '\0') ? g_strdup (checksum) : NULL);
-        }
-    }
-  else if (refs_to_fetch != NULL)
-    {
-      char **strviter = refs_to_fetch;
-      char **commitid_strviter = override_commit_ids ?: NULL;
-
-      while (*strviter)
-        {
-          const char *branch = *strviter;
-
-          if (ostree_validate_checksum_string (branch, NULL))
-            {
-              char *key = g_strdup (branch);
-              g_hash_table_add (commits_to_fetch, key);
-            }
-          else
-            {
-              if (!ostree_validate_rev (branch, error))
-                goto out;
-              char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
-              g_hash_table_insert (requested_refs_to_fetch,
-                                   ostree_collection_ref_new (NULL, branch), commitid);
-            }
-
-          strviter++;
-          if (commitid_strviter)
-            commitid_strviter++;
-        }
-    }
-  else
-    {
-      char **branches_iter;
-
-      branches_iter = configured_branches;
-
-      if (!(branches_iter && *branches_iter))
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "No configured branches for remote %s", remote_name_or_baseurl);
-          goto out;
-        }
-      for (;branches_iter && *branches_iter; branches_iter++)
-        {
-          const char *branch = *branches_iter;
-
-          g_hash_table_insert (requested_refs_to_fetch,
-                               ostree_collection_ref_new (NULL, branch), NULL);
-        }
-    }
 
   /* Resolve the checksum for each ref. This has to be done into a new hash table,
    * since we can’t modify the keys of @requested_refs_to_fetch while iterating
@@ -4900,6 +5130,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref);
+  g_clear_pointer (&pull_data->pending_fetch_delta_indexes, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref);
   g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL);

src/libostree/ostree-repo-static-delta-core.c

@@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo                  *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_list_static_delta_indexes:
+ * @self: Repo
+ * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * This function synchronously enumerates all static delta indexes in the
+ * repository, returning its result in @out_indexes.
+ *
+ * Since: 2020.7
+ */
+gboolean
+ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                       GPtrArray                  **out_indexes,
+                                       GCancellable                *cancellable,
+                                       GError                     **error)
+{
+  g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free);
+
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  gboolean exists;
+  if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter,
+                                     &exists, error))
+    return FALSE;
+  if (!exists)
+    {
+      /* Note early return */
+      ot_transfer_out_value (out_indexes, &ret_indexes);
+      return TRUE;
+    }
+
+  while (TRUE)
+    {
+      g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, };
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
+        return FALSE;
+      if (dent == NULL)
+        break;
+      if (dent->d_type != DT_DIR)
+        continue;
+      if (strlen (dent->d_name) != 2)
+        continue;
+
+      if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE,
+                                        &sub_dfd_iter, error))
+        return FALSE;
+
+      while (TRUE)
+        {
+          struct dirent *sub_dent;
+
+          if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent,
+                                                           cancellable, error))
+            return FALSE;
+          if (sub_dent == NULL)
+            break;
+          if (sub_dent->d_type != DT_REG)
+            continue;
+
+          const char *name1 = dent->d_name;
+          const char *name2 = sub_dent->d_name;
+
+          /* base64 len is 43, but 2 chars are in the parent dir name */
+          if (strlen (name2) != 41 + strlen(".index") ||
+              !g_str_has_suffix (name2, ".index"))
+            continue;
+
+          g_autoptr(GString) out = g_string_new (name1);
+          g_string_append_len (out, name2, 41);
+
+          char checksum[OSTREE_SHA256_STRING_LEN+1];
+          guchar csum[OSTREE_SHA256_DIGEST_LEN];
+
+          ostree_checksum_b64_inplace_to_bytes (out->str, csum);
+          ostree_checksum_inplace_from_bytes (csum, checksum);
+
+          g_ptr_array_add (ret_indexes, g_strdup (checksum));
+        }
+    }
+
+  ot_transfer_out_value (out_indexes, &ret_indexes);
+  return TRUE;
+}
+
 gboolean
 _ostree_repo_static_delta_part_have_all_objects (OstreeRepo             *repo,
                                                  GVariant               *checksum_array,
@@ -1118,3 +1205,207 @@ ostree_repo_static_delta_verify_signature (OstreeRepo       *self,
 
   return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error);
 }
+
+static void
+null_or_ptr_array_unref (GPtrArray *array)
+{
+  if (array != NULL)
+    g_ptr_array_unref (array);
+}
+
+static gboolean
+file_has_content (OstreeRepo   *repo,
+                  const char   *subpath,
+                  GBytes       *data,
+                  GCancellable *cancellable)
+{
+  struct stat stbuf;
+  glnx_autofd int existing_fd = -1;
+
+  if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL))
+    return FALSE;
+
+  if (stbuf.st_size != g_bytes_get_size (data))
+    return FALSE;
+
+  if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL))
+    return FALSE;
+
+  g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL);
+  if (existing_data == NULL)
+    return FALSE;
+
+  return g_bytes_equal (existing_data, data);
+}
+
+/**
+ * ostree_repo_static_delta_reindex:
+ * @repo: Repo
+ * @flags: Flags affecting the indexing operation
+ * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * The delta index for a particular commit lists all the existing deltas that can be used
+ * when downloading that commit. This operation regenerates these indexes, either for
+ * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that
+ * are reachable by an existing delta (if @opt_to_commit is %NULL).
+ *
+ * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary().
+ *
+ * Locking: shared
+ */
+gboolean
+ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                  OstreeStaticDeltaIndexFlags flags,
+                                  const char                 *opt_to_commit,
+                                  GCancellable               *cancellable,
+                                  GError                    **error)
+{
+  g_autoptr(GPtrArray) all_deltas = NULL;
+  g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */
+  gboolean opt_indexed_deltas;
+
+  /* Protect against parallel prune operation */
+  g_autoptr(OstreeRepoAutoLock) lock =
+    _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error);
+  if (!lock)
+    return FALSE;
+
+  /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */
+  if (!ot_keyfile_get_boolean_with_default (repo->config, "core",
+                                            "indexed-deltas", FALSE,
+                                            &opt_indexed_deltas, error))
+    return FALSE;
+
+  if (!opt_indexed_deltas)
+    {
+      g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo);
+      g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE);
+      if (!ostree_repo_write_config (repo, config, error))
+        return FALSE;
+    }
+
+  deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref);
+
+  if (opt_to_commit == NULL)
+    {
+      g_autoptr(GPtrArray) old_indexes = NULL;
+
+      /* To ensure all old index files either is regenerated, or
+       * removed, we initialize all existing indexes to NULL in the
+       * hashtable. */
+      if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error))
+        return FALSE;
+
+      for (int i = 0; i < old_indexes->len; i++)
+        {
+          const char *old_index = g_ptr_array_index (old_indexes, i);
+          g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL);
+        }
+    }
+  else
+    {
+      if (!ostree_validate_checksum_string (opt_to_commit, error))
+        return FALSE;
+
+      /* We ensure the specific old index either is regenerated, or removed */
+      g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL);
+    }
+
+  if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error))
+    return FALSE;
+
+  for (int i = 0; i < all_deltas->len; i++)
+    {
+      const char *delta_name = g_ptr_array_index (all_deltas, i);
+      g_autofree char *from = NULL;
+      g_autofree char *to = NULL;
+      GPtrArray *deltas_to_commit = NULL;
+
+      if (!_ostree_parse_delta_name (delta_name, &from, &to, error))
+        return FALSE;
+
+      if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0)
+        continue;
+
+      deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to);
+      if (deltas_to_commit == NULL)
+        {
+          deltas_to_commit = g_ptr_array_new_with_free_func (g_free);
+          g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit);
+        }
+
+      g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from));
+    }
+
+  GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms)
+    {
+      g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to);
+
+      if (froms == NULL)
+        {
+          /* No index to this checksum seen, delete if it exists */
+
+          g_debug ("Removing delta index for %s", to);
+          if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error))
+            return FALSE;
+        }
+      else
+        {
+          g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_autoptr(GVariant) index_variant = NULL;
+          g_autoptr(GBytes) index = NULL;
+
+          /* We sort on from here so that the index file is reproducible */
+          g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0);
+
+          g_variant_dict_init (&deltas_builder, NULL);
+
+          for (int i = 0; i < froms->len; i++)
+            {
+              const char *from = g_ptr_array_index (froms, i);
+              g_autofree char *delta_name = NULL;
+              GVariant *digest;
+
+              digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error);
+              if (digest == NULL)
+                return FALSE;
+
+              if (from != NULL)
+                delta_name = g_strconcat (from, "-", to, NULL);
+              else
+                delta_name = g_strdup (to);
+
+              g_variant_dict_insert_value (&deltas_builder, delta_name, digest);
+            }
+
+          /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when
+           * storing deltas in the summary. */
+          g_variant_dict_init (&index_builder, NULL);
+
+          g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+
+          index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder));
+          index = g_variant_get_data_as_bytes (index_variant);
+
+          g_autofree char *index_dirname = g_path_get_dirname (index_path);
+          if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error))
+            return FALSE;
+
+          /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will
+             lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */
+          if (file_has_content (repo, index_path, index, cancellable))
+            continue;
+
+          g_debug ("Updating delta index for %s", to);
+          if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path,
+                                              g_bytes_get_data (index, NULL), g_bytes_get_size (index),
+                                              0, cancellable, error))
+            return FALSE;
+        }
+    }
+
+  return TRUE;
+}

src/libostree/ostree-repo-static-delta-private.h

@@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo                 *repo,
                                   const char                 *delta_id,
                                   GCancellable               *cancellable,
                                   GError                    **error);
+gboolean
+_ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                   const char                 *opt_to_commit,
+                                   GCancellable               *cancellable,
+                                   GError                    **error);
 
 /* Used for static deltas which due to a historical mistake are
  * inconsistent endian.

src/libostree/ostree-repo.c

@@ -5222,6 +5222,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo     *self,
 #endif /* OSTREE_DISABLE_GPGME */
 }
 
+
+/**
+ * ostree_repo_gpg_sign_data:
+ * @self: Self
+ * @data: Data as a #GBytes
+ * @old_signatures: Existing signatures to append to (or %NULL)
+ * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys.
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @out_signature: (out): in case of success will contain signature
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Sign the given @data with the specified keys in @key_id. Similar to
+ * ostree_repo_add_gpg_signature_summary() but can be used on any
+ * data.
+ *
+ * You can use ostree_repo_gpg_verify_data() to verify the signatures.
+ *
+ * Returns: @TRUE if @data has been signed successfully,
+ * @FALSE in case of error (@error will contain the reason).
+ *
+ * Since: 2020.8
+ */
+gboolean
+ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                           GBytes         *data,
+                           GBytes         *old_signatures,
+                           const gchar   **key_id,
+                           const gchar    *homedir,
+                           GBytes        **out_signatures,
+                           GCancellable   *cancellable,
+                           GError        **error)
+{
+#ifndef OSTREE_DISABLE_GPGME
+  g_autoptr(GVariant) metadata = NULL;
+  g_autoptr(GVariant) res = NULL;
+
+  if (old_signatures)
+    metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE));
+
+  for (guint i = 0; key_id[i]; i++)
+    {
+      g_autoptr(GBytes) signature_data = NULL;
+      if (!sign_data (self, data, key_id[i], homedir,
+                      &signature_data,
+                      cancellable, error))
+        return FALSE;
+
+      g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
+      metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data);
+    }
+
+  res = g_variant_get_normal_form (metadata);
+  *out_signatures = g_variant_get_data_as_bytes (res);
+  return TRUE;
+#else
+  return glnx_throw (error, "GPG feature is disabled in a build time");
+#endif /* OSTREE_DISABLE_GPGME */
+}
+
+
 #ifndef OSTREE_DISABLE_GPGME
 /* Special remote for _ostree_repo_gpg_verify_with_metadata() */
 static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__";
@@ -5749,6 +5810,8 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
    * commits from working.
    */
   g_autoptr(OstreeRepoAutoLock) lock = NULL;
+  gboolean no_deltas_in_summary = FALSE;
+
   lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE,
                                       cancellable, error);
   if (!lock)
@@ -5781,6 +5844,12 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
       }
   }
 
+  if (!ot_keyfile_get_boolean_with_default (self->config, "core",
+                                            "no-deltas-in-summary", FALSE,
+                                            &no_deltas_in_summary, error))
+    return FALSE;
+
+  if (!no_deltas_in_summary)
     {
       g_autoptr(GPtrArray) delta_names = NULL;
       g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
@@ -5834,6 +5903,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
                                  g_variant_new_boolean (tombstone_commits));
   }
 
+  g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS,
+                               g_variant_new_boolean (TRUE));
+
   /* Add refs which have a collection specified, which could be in refs/mirrors,
    * refs/heads, and/or refs/remotes. */
   {
@@ -5927,6 +5999,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
     g_variant_ref_sink (summary);
   }
 
+  if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error))
+    return FALSE;
+
   if (!_ostree_repo_file_replace_contents (self,
                                            self->repo_dir_fd,
                                            "summary",

src/libostree/ostree-repo.h

@@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo                  *self,
                                               GCancellable                *cancellable,
                                               GError                     **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                                GPtrArray                  **out_indexes,
+                                                GCancellable                *cancellable,
+                                                GError                     **error);
+
 /**
  * OstreeStaticDeltaGenerateOpt:
  * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space
@@ -1068,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                             GCancellable                 *cancellable,
                                             GError                      **error);
 
+/**
+ * OstreeStaticDeltaIndexFlags:
+ * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags
+ *
+ * Flags controlling static delta index generation.
+ */
+typedef enum {
+  OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
+} OstreeStaticDeltaIndexFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_static_delta_reindex (OstreeRepo                  *repo,
+                                           OstreeStaticDeltaIndexFlags flags,
+                                           const char                  *opt_to_commit,
+                                           GCancellable                *cancellable,
+                                           GError                     **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo   *self,
                                                                   GFile        *dir_or_file,
@@ -1393,6 +1416,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo     *self,
                                            GCancellable   *cancellable,
                                            GError        **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                                    GBytes         *data,
+                                    GBytes         *old_signatures,
+                                    const gchar   **key_id,
+                                    const gchar    *homedir,
+                                    GBytes        **out_signatures,
+                                    GCancellable   *cancellable,
+                                    GError        **error);
+
 _OSTREE_PUBLIC
 OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo    *self,
                                                        const gchar   *commit_checksum,

src/ostree/ot-builtin-static-delta.c

@@ -53,6 +53,8 @@ BUILTINPROTO(delete);
 BUILTINPROTO(generate);
 BUILTINPROTO(apply_offline);
 BUILTINPROTO(verify);
+BUILTINPROTO(indexes);
+BUILTINPROTO(reindex);
 
 #undef BUILTINPROTO
 
@@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = {
   { "verify", OSTREE_BUILTIN_FLAG_NONE,
     ot_static_delta_builtin_verify,
     "Verify static delta signatures" },
+  { "indexes", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_indexes,
+    "List static delta indexes" },
+  { "reindex", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_reindex,
+    "Regenerate static delta indexes" },
   { NULL, 0, NULL, NULL }
 };
 
@@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = {
   { NULL }
 };
 
+static GOptionEntry indexes_options[] = {
+  { NULL }
+};
+
+static GOptionEntry reindex_options[] = {
+  { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" },
+  { NULL }
+};
+
 static void
 static_delta_usage (char    **argv,
                     gboolean  is_error)
@@ -176,6 +193,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in
   return TRUE;
 }
 
+static gboolean
+ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(OstreeRepo) repo = NULL;
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+  if (!ostree_option_context_parse (context, indexes_options, &argc, &argv,
+                                    invocation, &repo, cancellable, error))
+    return FALSE;
+
+  g_autoptr(GPtrArray) indexes = NULL;
+  if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error))
+    return FALSE;
+
+  if (indexes->len == 0)
+    g_print ("(No static deltas indexes)\n");
+  else
+    {
+      for (guint i = 0; i < indexes->len; i++)
+        g_print ("%s\n", (char*)indexes->pdata[i]);
+    }
+
+  return TRUE;
+}
+
+static gboolean
+ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+
+  g_autoptr(OstreeRepo) repo = NULL;
+  if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error))
+    return FALSE;
+
+  if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+
 static gboolean
 ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {

tests/pull-test.sh

@@ -55,10 +55,10 @@ function verify_initial_contents() {
 }
 
 if has_gpgme; then
-    echo "1..35"
+    echo "1..36"
 else
     # 3 tests needs GPG support
-    echo "1..32"
+    echo "1..33"
 fi
 
 # Try both syntaxes
@@ -381,6 +381,25 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older"
 ${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev}
 echo "ok pull timestamp checking"
 
+# test pull without override commit use summary, but with doesn't use summary
+# We temporarily replace summary with broken one to detect if it is used
+mv ostree-srv/gnomerepo/summary ostree-srv/gnomerepo/summary.backup
+echo "broken" > ostree-srv/gnomerepo/summary
+
+repo_init --no-sign-verify
+rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
+# This will need summary, so will fail
+if ${CMD_PREFIX} ostree --repo=repo -v pull origin main; then
+  assert_not_reached "Should have failed with broken summary"
+fi
+# This won't need summary so will not fail
+${CMD_PREFIX} ostree --repo=repo pull origin main@${rev}
+
+# Restore summary
+mv ostree-srv/gnomerepo/summary.backup ostree-srv/gnomerepo/summary
+
+echo "ok pull with override id doesn't use summary"
+
 cd ${test_tmpdir}
 repo_init --no-sign-verify
 ${CMD_PREFIX} ostree --repo=repo pull origin main

tests/test-delta.sh

@@ -28,7 +28,7 @@ skip_without_user_xattrs
 bindatafiles="bash true ostree"
 morebindatafiles="false ls"
 
-echo '1..12'
+echo '1..13'
 
 mkdir repo
 ostree_repo_init repo --mode=archive
@@ -90,6 +90,11 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
 ${CMD_PREFIX} ostree --repo=repo prune
 ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
 
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^1$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+
 permuteDirectory 1 files
 ${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files
 
@@ -119,6 +124,12 @@ ${CMD_PREFIX} ostree --repo=repo static-delta generate --max-bsdiff-size=10000 -
 
 ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1
 
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^2$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${newrev} || exit 1
+
 if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --empty 2>>err.txt; then
     assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded"
 fi
@@ -249,6 +260,41 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
 
 echo 'ok pull empty delta part'
 
+rm -rf repo/delta-indexes
+${CMD_PREFIX} ostree --repo=repo summary -u
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary true
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_not_file_has_content summary.txt "ostree\.static\-deltas"
+
+rm -rf repo2
+mkdir repo2 && ostree_repo_init repo2 --mode=bare-user
+${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
+
+rm -rf repo/delta-indexes
+if ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} &> no-delta.txt; then
+    assert_not_reached "failing pull with  --require-static-deltas unexpectedly succeeded"
+fi
+assert_file_has_content no-delta.txt "Static deltas required, but none found for"
+
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev}
+
+${CMD_PREFIX} ostree --repo=repo2 fsck
+${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary false
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+echo 'ok pull delta part with delta index'
+
 # Make a new branch to test "rebase deltas"
 echo otherbranch-content > files/otherbranch-content
 ${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files