Merge pull request #2188 from alexlarsson/delta-indexes

Add indexes for deltas outside of the summary

Makefile-libostree.am

@@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \
 endif # USE_GPGME
 
 symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
-#if BUILDOPT_IS_DEVEL_BUILD
+if BUILDOPT_IS_DEVEL_BUILD
-#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
-#endif
+endif
 # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
 wl_versionscript_arg = -Wl,--version-script=
 EXTRA_DIST += \

apidoc/ostree-sections.txt

@@ -412,6 +412,8 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE
 ostree_repo_list_objects
 ostree_repo_list_commit_objects_starting_with
 ostree_repo_list_static_delta_names
+ostree_repo_list_static_delta_indexes
+ostree_repo_static_delta_reindex
 OstreeStaticDeltaGenerateOpt
 ostree_repo_static_delta_generate
 ostree_repo_static_delta_execute_offline_with_signature
@@ -445,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed
 ostree_repo_sign_commit
 ostree_repo_append_gpg_signature
 ostree_repo_add_gpg_signature_summary
+ostree_repo_gpg_sign_data
 ostree_repo_gpg_verify_data
 ostree_repo_verify_commit
 ostree_repo_verify_commit_ext

man/ostree.repo-config.xml

@@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA.
         costly).
         </para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>no-deltas-in-summary</varname></term>
+        <listitem><para>Boolean value controlling whether OSTree should skip
+        putting an index of available deltas in the summary file. Defaults to false.
+        </para>
+        <para>
+        Since 2020.7 OSTree can use delta indexes outside the summary file,
+        making the summary file smaller (especially for larger repositories). However
+        by default we still create the index in the summary file to make older clients
+        work. If you know all clients will be 2020.7 later you can enable this to
+        save network bandwidth.
+        </para></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

src/libostree/libostree-devel.sym

@@ -17,9 +17,12 @@
   Boston, MA 02111-1307, USA.
 ***/
 
-/* Copy the bits below and uncomment the include in Makefile-libostree.am
+LIBOSTREE_2020.8 {
-   when adding a symbol.
+global:
-*/
+  ostree_repo_list_static_delta_indexes;
+  ostree_repo_static_delta_reindex;
+  ostree_repo_gpg_sign_data;
+} LIBOSTREE_2020.7;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste

src/libostree/ostree-core-private.h

@@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
                                              const char        *to,
                                              guint              i);
 
+char *
+_ostree_get_relative_static_delta_index_path (const char        *to);
+
 static inline char * _ostree_get_commitpartial_path (const char *checksum)
 {
   return g_strconcat ("state/", checksum, ".commitpartial", NULL);

src/libostree/ostree-core.c

@@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char         *checksum,
   return g_string_free (path, FALSE);
 }
 
-char *
+static GString *
-_ostree_get_relative_static_delta_path (const char *from,
+static_delta_path_base (const char *dir,
-                                        const char *to,
+                        const char *from,
-                                        const char *target)
+                        const char *to)
 {
   guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
   char to_b64[44];
   guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
-  GString *ret = g_string_new ("deltas/");
+  GString *ret = g_string_new (dir);
 
   ostree_checksum_inplace_to_bytes (to, csum_to);
   ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
@@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from,
     g_string_append_c (ret, '/');
   g_string_append (ret, to_b64 + 2);
 
+  return ret;
+}
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+                                        const char *to,
+                                        const char *target)
+{
+  GString *ret = static_delta_path_base ("deltas/", from, to);
+
   if (target != NULL)
     {
       g_string_append_c (ret, '/');
@@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
   return _ostree_get_relative_static_delta_path (from, to, partstr);
 }
 
+char *
+_ostree_get_relative_static_delta_index_path (const char *to)
+{
+  GString *ret = static_delta_path_base ("delta-indexes/", NULL, to);
+
+  g_string_append (ret, ".index");
+
+  return g_string_free (ret, FALSE);
+}
+
 gboolean
 _ostree_parse_delta_name (const char   *delta_name,
                           char        **out_from,

src/libostree/ostree-repo-private.h

@@ -57,6 +57,7 @@ G_BEGIN_DECLS
 #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map"
 #define OSTREE_SUMMARY_MODE "ostree.summary.mode"
 #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits"
+#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas"
 
 #define _OSTREE_PAYLOAD_LINK_PREFIX "../"
 #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1)

src/libostree/ostree-repo-pull-private.h

@@ -78,7 +78,9 @@ typedef struct {
   char             *summary_sig_etag;
   guint64           summary_sig_last_modified;  /* seconds since the epoch */
   GVariant         *summary;
-  GHashTable       *summary_deltas_checksums;
+  GHashTable       *summary_deltas_checksums; /* Filled from summary and delta indexes */
+  gboolean          summary_has_deltas; /* True if the summary existed and had a delta index */
+  gboolean          has_indexed_deltas;
   GHashTable       *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
   GHashTable       *verified_commits; /* Set<checksum> of commits that have been verified */
   GHashTable       *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */
@@ -93,6 +95,7 @@ typedef struct {
   GHashTable       *requested_fallback_content; /* Maps checksum to itself */
   GHashTable       *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
   GHashTable       *pending_fetch_content; /* Map<checksum,FetchObjectData> */
+  GHashTable       *pending_fetch_delta_indexes; /* Set<FetchDeltaIndexData> */
   GHashTable       *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
   GHashTable       *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
   guint             n_outstanding_metadata_fetches;

src/libostree/ostree-repo-static-delta-core.c

@@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo                  *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_list_static_delta_indexes:
+ * @self: Repo
+ * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * This function synchronously enumerates all static delta indexes in the
+ * repository, returning its result in @out_indexes.
+ *
+ * Since: 2020.7
+ */
+gboolean
+ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                       GPtrArray                  **out_indexes,
+                                       GCancellable                *cancellable,
+                                       GError                     **error)
+{
+  g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free);
+
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  gboolean exists;
+  if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter,
+                                     &exists, error))
+    return FALSE;
+  if (!exists)
+    {
+      /* Note early return */
+      ot_transfer_out_value (out_indexes, &ret_indexes);
+      return TRUE;
+    }
+
+  while (TRUE)
+    {
+      g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, };
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
+        return FALSE;
+      if (dent == NULL)
+        break;
+      if (dent->d_type != DT_DIR)
+        continue;
+      if (strlen (dent->d_name) != 2)
+        continue;
+
+      if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE,
+                                        &sub_dfd_iter, error))
+        return FALSE;
+
+      while (TRUE)
+        {
+          struct dirent *sub_dent;
+
+          if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent,
+                                                           cancellable, error))
+            return FALSE;
+          if (sub_dent == NULL)
+            break;
+          if (sub_dent->d_type != DT_REG)
+            continue;
+
+          const char *name1 = dent->d_name;
+          const char *name2 = sub_dent->d_name;
+
+          /* base64 len is 43, but 2 chars are in the parent dir name */
+          if (strlen (name2) != 41 + strlen(".index") ||
+              !g_str_has_suffix (name2, ".index"))
+            continue;
+
+          g_autoptr(GString) out = g_string_new (name1);
+          g_string_append_len (out, name2, 41);
+
+          char checksum[OSTREE_SHA256_STRING_LEN+1];
+          guchar csum[OSTREE_SHA256_DIGEST_LEN];
+
+          ostree_checksum_b64_inplace_to_bytes (out->str, csum);
+          ostree_checksum_inplace_from_bytes (csum, checksum);
+
+          g_ptr_array_add (ret_indexes, g_strdup (checksum));
+        }
+    }
+
+  ot_transfer_out_value (out_indexes, &ret_indexes);
+  return TRUE;
+}
+
 gboolean
 _ostree_repo_static_delta_part_have_all_objects (OstreeRepo             *repo,
                                                  GVariant               *checksum_array,
@@ -1118,3 +1205,207 @@ ostree_repo_static_delta_verify_signature (OstreeRepo       *self,
 
   return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error);
 }
+
+static void
+null_or_ptr_array_unref (GPtrArray *array)
+{
+  if (array != NULL)
+    g_ptr_array_unref (array);
+}
+
+static gboolean
+file_has_content (OstreeRepo   *repo,
+                  const char   *subpath,
+                  GBytes       *data,
+                  GCancellable *cancellable)
+{
+  struct stat stbuf;
+  glnx_autofd int existing_fd = -1;
+
+  if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL))
+    return FALSE;
+
+  if (stbuf.st_size != g_bytes_get_size (data))
+    return FALSE;
+
+  if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL))
+    return FALSE;
+
+  g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL);
+  if (existing_data == NULL)
+    return FALSE;
+
+  return g_bytes_equal (existing_data, data);
+}
+
+/**
+ * ostree_repo_static_delta_reindex:
+ * @repo: Repo
+ * @flags: Flags affecting the indexing operation
+ * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * The delta index for a particular commit lists all the existing deltas that can be used
+ * when downloading that commit. This operation regenerates these indexes, either for
+ * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that
+ * are reachable by an existing delta (if @opt_to_commit is %NULL).
+ *
+ * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary().
+ *
+ * Locking: shared
+ */
+gboolean
+ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                  OstreeStaticDeltaIndexFlags flags,
+                                  const char                 *opt_to_commit,
+                                  GCancellable               *cancellable,
+                                  GError                    **error)
+{
+  g_autoptr(GPtrArray) all_deltas = NULL;
+  g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */
+  gboolean opt_indexed_deltas;
+
+  /* Protect against parallel prune operation */
+  g_autoptr(OstreeRepoAutoLock) lock =
+    _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error);
+  if (!lock)
+    return FALSE;
+
+  /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */
+  if (!ot_keyfile_get_boolean_with_default (repo->config, "core",
+                                            "indexed-deltas", FALSE,
+                                            &opt_indexed_deltas, error))
+    return FALSE;
+
+  if (!opt_indexed_deltas)
+    {
+      g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo);
+      g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE);
+      if (!ostree_repo_write_config (repo, config, error))
+        return FALSE;
+    }
+
+  deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref);
+
+  if (opt_to_commit == NULL)
+    {
+      g_autoptr(GPtrArray) old_indexes = NULL;
+
+      /* To ensure all old index files either is regenerated, or
+       * removed, we initialize all existing indexes to NULL in the
+       * hashtable. */
+      if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error))
+        return FALSE;
+
+      for (int i = 0; i < old_indexes->len; i++)
+        {
+          const char *old_index = g_ptr_array_index (old_indexes, i);
+          g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL);
+        }
+    }
+  else
+    {
+      if (!ostree_validate_checksum_string (opt_to_commit, error))
+        return FALSE;
+
+      /* We ensure the specific old index either is regenerated, or removed */
+      g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL);
+    }
+
+  if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error))
+    return FALSE;
+
+  for (int i = 0; i < all_deltas->len; i++)
+    {
+      const char *delta_name = g_ptr_array_index (all_deltas, i);
+      g_autofree char *from = NULL;
+      g_autofree char *to = NULL;
+      GPtrArray *deltas_to_commit = NULL;
+
+      if (!_ostree_parse_delta_name (delta_name, &from, &to, error))
+        return FALSE;
+
+      if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0)
+        continue;
+
+      deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to);
+      if (deltas_to_commit == NULL)
+        {
+          deltas_to_commit = g_ptr_array_new_with_free_func (g_free);
+          g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit);
+        }
+
+      g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from));
+    }
+
+  GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms)
+    {
+      g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to);
+
+      if (froms == NULL)
+        {
+          /* No index to this checksum seen, delete if it exists */
+
+          g_debug ("Removing delta index for %s", to);
+          if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error))
+            return FALSE;
+        }
+      else
+        {
+          g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_autoptr(GVariant) index_variant = NULL;
+          g_autoptr(GBytes) index = NULL;
+
+          /* We sort on from here so that the index file is reproducible */
+          g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0);
+
+          g_variant_dict_init (&deltas_builder, NULL);
+
+          for (int i = 0; i < froms->len; i++)
+            {
+              const char *from = g_ptr_array_index (froms, i);
+              g_autofree char *delta_name = NULL;
+              GVariant *digest;
+
+              digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error);
+              if (digest == NULL)
+                return FALSE;
+
+              if (from != NULL)
+                delta_name = g_strconcat (from, "-", to, NULL);
+              else
+                delta_name = g_strdup (to);
+
+              g_variant_dict_insert_value (&deltas_builder, delta_name, digest);
+            }
+
+          /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when
+           * storing deltas in the summary. */
+          g_variant_dict_init (&index_builder, NULL);
+
+          g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+
+          index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder));
+          index = g_variant_get_data_as_bytes (index_variant);
+
+          g_autofree char *index_dirname = g_path_get_dirname (index_path);
+          if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error))
+            return FALSE;
+
+          /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will
+             lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */
+          if (file_has_content (repo, index_path, index, cancellable))
+            continue;
+
+          g_debug ("Updating delta index for %s", to);
+          if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path,
+                                              g_bytes_get_data (index, NULL), g_bytes_get_size (index),
+                                              0, cancellable, error))
+            return FALSE;
+        }
+    }
+
+  return TRUE;
+}

src/libostree/ostree-repo-static-delta-private.h

@@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo                 *repo,
                                   const char                 *delta_id,
                                   GCancellable               *cancellable,
                                   GError                    **error);
+gboolean
+_ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                   const char                 *opt_to_commit,
+                                   GCancellable               *cancellable,
+                                   GError                    **error);
 
 /* Used for static deltas which due to a historical mistake are
  * inconsistent endian.

src/libostree/ostree-repo.c

@@ -5222,6 +5222,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo     *self,
 #endif /* OSTREE_DISABLE_GPGME */
 }
 
+
+/**
+ * ostree_repo_gpg_sign_data:
+ * @self: Self
+ * @data: Data as a #GBytes
+ * @old_signatures: Existing signatures to append to (or %NULL)
+ * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys.
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @out_signature: (out): in case of success will contain signature
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Sign the given @data with the specified keys in @key_id. Similar to
+ * ostree_repo_add_gpg_signature_summary() but can be used on any
+ * data.
+ *
+ * You can use ostree_repo_gpg_verify_data() to verify the signatures.
+ *
+ * Returns: @TRUE if @data has been signed successfully,
+ * @FALSE in case of error (@error will contain the reason).
+ *
+ * Since: 2020.8
+ */
+gboolean
+ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                           GBytes         *data,
+                           GBytes         *old_signatures,
+                           const gchar   **key_id,
+                           const gchar    *homedir,
+                           GBytes        **out_signatures,
+                           GCancellable   *cancellable,
+                           GError        **error)
+{
+#ifndef OSTREE_DISABLE_GPGME
+  g_autoptr(GVariant) metadata = NULL;
+  g_autoptr(GVariant) res = NULL;
+
+  if (old_signatures)
+    metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE));
+
+  for (guint i = 0; key_id[i]; i++)
+    {
+      g_autoptr(GBytes) signature_data = NULL;
+      if (!sign_data (self, data, key_id[i], homedir,
+                      &signature_data,
+                      cancellable, error))
+        return FALSE;
+
+      g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
+      metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data);
+    }
+
+  res = g_variant_get_normal_form (metadata);
+  *out_signatures = g_variant_get_data_as_bytes (res);
+  return TRUE;
+#else
+  return glnx_throw (error, "GPG feature is disabled in a build time");
+#endif /* OSTREE_DISABLE_GPGME */
+}
+
+
 #ifndef OSTREE_DISABLE_GPGME
 /* Special remote for _ostree_repo_gpg_verify_with_metadata() */
 static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__";
@@ -5749,6 +5810,8 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
    * commits from working.
    */
   g_autoptr(OstreeRepoAutoLock) lock = NULL;
+  gboolean no_deltas_in_summary = FALSE;
+
   lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE,
                                       cancellable, error);
   if (!lock)
@@ -5781,35 +5844,41 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
       }
   }
 
-  {
+  if (!ot_keyfile_get_boolean_with_default (self->config, "core",
-    g_autoptr(GPtrArray) delta_names = NULL;
+                                            "no-deltas-in-summary", FALSE,
-    g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+                                            &no_deltas_in_summary, error))
+    return FALSE;
 
-    if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
+  if (!no_deltas_in_summary)
-      return FALSE;
+    {
+      g_autoptr(GPtrArray) delta_names = NULL;
+      g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
 
-    g_variant_dict_init (&deltas_builder, NULL);
+      if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
-    for (guint i = 0; i < delta_names->len; i++)
+        return FALSE;
-      {
-        g_autofree char *from = NULL;
-        g_autofree char *to = NULL;
-        GVariant *digest;
 
-        if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
+      g_variant_dict_init (&deltas_builder, NULL);
-          return FALSE;
+      for (guint i = 0; i < delta_names->len; i++)
+        {
+          g_autofree char *from = NULL;
+          g_autofree char *to = NULL;
+          GVariant *digest;
 
-        digest = _ostree_repo_static_delta_superblock_digest (self,
+          if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
-                                                              (from && from[0]) ? from : NULL,
+            return FALSE;
-                                                              to, cancellable, error);
-        if (digest == NULL)
-          return FALSE;
 
-        g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
+          digest = _ostree_repo_static_delta_superblock_digest (self,
-      }
+                                                                (from && from[0]) ? from : NULL,
+                                                                to, cancellable, error);
+          if (digest == NULL)
+            return FALSE;
 
-    if (delta_names->len > 0)
+          g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
-      g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+        }
-  }
+
+      if (delta_names->len > 0)
+        g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+    }
 
   {
     g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED,
@@ -5834,6 +5903,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
                                  g_variant_new_boolean (tombstone_commits));
   }
 
+  g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS,
+                               g_variant_new_boolean (TRUE));
+
   /* Add refs which have a collection specified, which could be in refs/mirrors,
    * refs/heads, and/or refs/remotes. */
   {
@@ -5927,6 +5999,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
     g_variant_ref_sink (summary);
   }
 
+  if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error))
+    return FALSE;
+
   if (!_ostree_repo_file_replace_contents (self,
                                            self->repo_dir_fd,
                                            "summary",

src/libostree/ostree-repo.h

@@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo                  *self,
                                               GCancellable                *cancellable,
                                               GError                     **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                                GPtrArray                  **out_indexes,
+                                                GCancellable                *cancellable,
+                                                GError                     **error);
+
 /**
  * OstreeStaticDeltaGenerateOpt:
  * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space
@@ -1068,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                             GCancellable                 *cancellable,
                                             GError                      **error);
 
+/**
+ * OstreeStaticDeltaIndexFlags:
+ * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags
+ *
+ * Flags controlling static delta index generation.
+ */
+typedef enum {
+  OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
+} OstreeStaticDeltaIndexFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_static_delta_reindex (OstreeRepo                  *repo,
+                                           OstreeStaticDeltaIndexFlags flags,
+                                           const char                  *opt_to_commit,
+                                           GCancellable                *cancellable,
+                                           GError                     **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo   *self,
                                                                   GFile        *dir_or_file,
@@ -1393,6 +1416,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo     *self,
                                            GCancellable   *cancellable,
                                            GError        **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                                    GBytes         *data,
+                                    GBytes         *old_signatures,
+                                    const gchar   **key_id,
+                                    const gchar    *homedir,
+                                    GBytes        **out_signatures,
+                                    GCancellable   *cancellable,
+                                    GError        **error);
+
 _OSTREE_PUBLIC
 OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo    *self,
                                                        const gchar   *commit_checksum,

src/ostree/ot-builtin-static-delta.c

@@ -53,6 +53,8 @@ BUILTINPROTO(delete);
 BUILTINPROTO(generate);
 BUILTINPROTO(apply_offline);
 BUILTINPROTO(verify);
+BUILTINPROTO(indexes);
+BUILTINPROTO(reindex);
 
 #undef BUILTINPROTO
 
@@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = {
   { "verify", OSTREE_BUILTIN_FLAG_NONE,
     ot_static_delta_builtin_verify,
     "Verify static delta signatures" },
+  { "indexes", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_indexes,
+    "List static delta indexes" },
+  { "reindex", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_reindex,
+    "Regenerate static delta indexes" },
   { NULL, 0, NULL, NULL }
 };
 
@@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = {
   { NULL }
 };
 
+static GOptionEntry indexes_options[] = {
+  { NULL }
+};
+
+static GOptionEntry reindex_options[] = {
+  { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" },
+  { NULL }
+};
+
 static void
 static_delta_usage (char    **argv,
                     gboolean  is_error)
@@ -176,6 +193,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in
   return TRUE;
 }
 
+static gboolean
+ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(OstreeRepo) repo = NULL;
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+  if (!ostree_option_context_parse (context, indexes_options, &argc, &argv,
+                                    invocation, &repo, cancellable, error))
+    return FALSE;
+
+  g_autoptr(GPtrArray) indexes = NULL;
+  if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error))
+    return FALSE;
+
+  if (indexes->len == 0)
+    g_print ("(No static deltas indexes)\n");
+  else
+    {
+      for (guint i = 0; i < indexes->len; i++)
+        g_print ("%s\n", (char*)indexes->pdata[i]);
+    }
+
+  return TRUE;
+}
+
+static gboolean
+ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+
+  g_autoptr(OstreeRepo) repo = NULL;
+  if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error))
+    return FALSE;
+
+  if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+
 static gboolean
 ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {

tests/pull-test.sh

@@ -55,10 +55,10 @@ function verify_initial_contents() {
 }
 
 if has_gpgme; then
-    echo "1..35"
+    echo "1..36"
 else
     # 3 tests needs GPG support
-    echo "1..32"
+    echo "1..33"
 fi
 
 # Try both syntaxes
@@ -381,6 +381,25 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older"
 ${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev}
 echo "ok pull timestamp checking"
 
+# test pull without override commit use summary, but with doesn't use summary
+# We temporarily replace summary with broken one to detect if it is used
+mv ostree-srv/gnomerepo/summary ostree-srv/gnomerepo/summary.backup
+echo "broken" > ostree-srv/gnomerepo/summary
+
+repo_init --no-sign-verify
+rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
+# This will need summary, so will fail
+if ${CMD_PREFIX} ostree --repo=repo -v pull origin main; then
+  assert_not_reached "Should have failed with broken summary"
+fi
+# This won't need summary so will not fail
+${CMD_PREFIX} ostree --repo=repo pull origin main@${rev}
+
+# Restore summary
+mv ostree-srv/gnomerepo/summary.backup ostree-srv/gnomerepo/summary
+
+echo "ok pull with override id doesn't use summary"
+
 cd ${test_tmpdir}
 repo_init --no-sign-verify
 ${CMD_PREFIX} ostree --repo=repo pull origin main

tests/test-delta.sh

@@ -28,7 +28,7 @@ skip_without_user_xattrs
 bindatafiles="bash true ostree"
 morebindatafiles="false ls"
 
-echo '1..12'
+echo '1..13'
 
 mkdir repo
 ostree_repo_init repo --mode=archive
@@ -90,6 +90,11 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
 ${CMD_PREFIX} ostree --repo=repo prune
 ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1
 
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^1$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+
 permuteDirectory 1 files
 ${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files
 
@@ -119,6 +124,12 @@ ${CMD_PREFIX} ostree --repo=repo static-delta generate --max-bsdiff-size=10000 -
 
 ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1
 
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount
+assert_file_has_content indexcount "^2$"
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1
+${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${newrev} || exit 1
+
 if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --empty 2>>err.txt; then
     assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded"
 fi
@@ -249,6 +260,41 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
 
 echo 'ok pull empty delta part'
 
+rm -rf repo/delta-indexes
+${CMD_PREFIX} ostree --repo=repo summary -u
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary true
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_not_file_has_content summary.txt "ostree\.static\-deltas"
+
+rm -rf repo2
+mkdir repo2 && ostree_repo_init repo2 --mode=bare-user
+${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
+
+rm -rf repo/delta-indexes
+if ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} &> no-delta.txt; then
+    assert_not_reached "failing pull with  --require-static-deltas unexpectedly succeeded"
+fi
+assert_file_has_content no-delta.txt "Static deltas required, but none found for"
+
+${CMD_PREFIX} ostree --repo=repo static-delta reindex
+${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev}
+
+${CMD_PREFIX} ostree --repo=repo2 fsck
+${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null
+
+${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary false
+${CMD_PREFIX} ostree --repo=repo summary -u
+
+${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt
+assert_file_has_content summary.txt "ostree\.static\-deltas"
+
+echo 'ok pull delta part with delta index'
+
 # Make a new branch to test "rebase deltas"
 echo otherbranch-content > files/otherbranch-content
 ${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files