When mirroring, write content directly, do not verify

When doing a pull --mirror from an archive-z2 repository into another
archive-z2 repository, currently we gunzip/checksum/gzip each content
object.  The re-gzip process in particular is fairly expensive.

This does assume that the upstream content is trusted and correct.
It'd be nice in the future to do at least a CRC check, if not the full
checksum.  (Could we append CRC data to the end of filez objects?)

We could also choose to only do this optimization if fetching over
TLS.

before: 1626 metadata, 20320 content objects fetched; 299634 KiB transferred in 62 seconds
after : 1626 metadata, 20320 content objects fetched; 299634 KiB transferred in 11 seconds

src/libostree/ostree-repo-commit.c

@@ -123,12 +123,62 @@ write_file_metadata_to_xattr (int fd,
   return TRUE;
 }
 
+gboolean
+_ostree_repo_commit_loose_final (OstreeRepo        *self,
+                                 const char        *checksum,
+                                 OstreeObjectType   objtype,
+                                 int                temp_dfd,
+                                 const char        *temp_filename,
+                                 GCancellable      *cancellable,
+                                 GError           **error)
+{
+  gboolean ret = FALSE;
+  gs_free gchar *tmp_dest = NULL;
+  int dest_dfd;
+  char tmpbuf[_OSTREE_LOOSE_PATH_MAX];
+  const char *dest;
+
+  if (self->in_transaction)
+    {
+      _ostree_repo_get_tmpobject_path (self, tmpbuf, checksum, objtype);
+      tmp_dest = g_strdup (tmpbuf);
+      dest_dfd = self->tmp_dir_fd;
+      dest = tmp_dest;
+    }
+  else
+    {
+      _ostree_loose_path (tmpbuf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
+      
+      if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, tmpbuf,
+                                                cancellable, error))
+        goto out;
+
+      dest_dfd = self->objects_dir_fd;
+      dest = tmpbuf;
+    }
+
+  if (G_UNLIKELY (renameat (temp_dfd, temp_filename,
+                            dest_dfd, dest) == -1))
+    {
+      if (errno != EEXIST)
+        {
+          gs_set_error_from_errno (error, errno);
+          g_prefix_error (error, "Storing file '%s': ", temp_filename);
+          goto out;
+        }
+      else
+        (void) unlinkat (temp_dfd, temp_filename, 0);
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
 
 static gboolean
 commit_loose_object_trusted (OstreeRepo        *self,
                              const char        *checksum,
                              OstreeObjectType   objtype,
-                             const char        *loose_path,
                              const char        *temp_filename,
                              gboolean           object_is_symlink,
                              guint32            uid,
@@ -269,42 +319,11 @@ commit_loose_object_trusted (OstreeRepo        *self,
         }
     }
 
-  if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path,
+  if (!_ostree_repo_commit_loose_final (self, checksum, objtype,
+                                        self->tmp_dir_fd, temp_filename,
                                         cancellable, error))
     goto out;
   
-  {
-    gs_free gchar *tmp_dest = NULL;
-    int dir;
-    const char *dest;
-
-    if (self->in_transaction)
-      {
-        char tmpbuf[_OSTREE_LOOSE_PATH_MAX];
-        _ostree_repo_get_tmpobject_path (self, tmpbuf, checksum, objtype);
-        tmp_dest = g_strdup (tmpbuf);
-        dir = self->tmp_dir_fd;
-        dest = tmp_dest;
-      }
-    else
-      {
-        dir = self->objects_dir_fd;
-        dest = loose_path;
-      }
-
-    if (G_UNLIKELY (renameat (self->tmp_dir_fd, temp_filename,
-                              dir, dest) == -1))
-      {
-        if (errno != EEXIST)
-          {
-            gs_set_error_from_errno (error, errno);
-            g_prefix_error (error, "Storing file '%s': ", temp_filename);
-            goto out;
-          }
-        else
-          (void) unlinkat (self->tmp_dir_fd, temp_filename, 0);
-      }
-  }
   ret = TRUE;
  out:
   return ret;
@@ -515,14 +534,10 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
                                           GError             **error)
 {
   gboolean ret = FALSE;
-  char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
 
   if (state->fd != -1)
     {
-      _ostree_loose_path (loose_objpath, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
-      
       if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE,
-                                        loose_objpath,
                                         state->temp_filename,
                                         FALSE, uid, gid, mode,
                                         xattrs, state->fd,
@@ -778,7 +793,6 @@ write_object (OstreeRepo         *self,
         fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
       
       if (!commit_loose_object_trusted (self, actual_checksum, objtype,
-                                        loose_objpath,
                                         temp_filename,
                                         object_is_symlink,
                                         uid, gid, mode,

src/libostree/ostree-repo-private.h

@@ -194,6 +194,15 @@ _ostree_repo_gpg_verify_file_with_metadata (OstreeRepo          *self,
                                             GCancellable        *cancellable,
                                             GError             **error);
 
+gboolean
+_ostree_repo_commit_loose_final (OstreeRepo        *self,
+                                 const char        *checksum,
+                                 OstreeObjectType   objtype,
+                                 int                temp_dfd,
+                                 const char        *temp_filename,
+                                 GCancellable      *cancellable,
+                                 GError           **error);
+
 typedef struct {
   int fd;
   char *temp_filename;

src/libostree/ostree-repo-pull.c

@@ -82,6 +82,8 @@ typedef struct {
   int               maxdepth;
   guint64           start_time;
 
+  gboolean          is_mirror;
+
   char         *dir;
   gboolean      commitpartial_exists;
 
@@ -597,6 +599,27 @@ content_fetch_on_complete (GObject        *object,
 
   g_debug ("fetch of %s complete", ostree_object_to_string (checksum, objtype));
 
+  if (pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2)
+    {
+      gboolean have_object;
+      if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,
+                                   &have_object,
+                                   cancellable, error))
+        goto out;
+
+      if (!have_object)
+        {
+          if (!_ostree_repo_commit_loose_final (pull_data->repo, checksum, OSTREE_OBJECT_TYPE_FILE,
+                                                pull_data->tmpdir_dfd, temp_path,
+                                                cancellable, error))
+            goto out;
+        }
+      pull_data->n_fetched_content++;
+    }
+  else
+    {
+      /* Non-mirroring path */
+      
       if (!ostree_content_file_parse_at (TRUE, pull_data->tmpdir_dfd, temp_path, FALSE,
                                          &file_in, &file_info, &xattrs,
                                          cancellable, error))
@@ -622,6 +645,7 @@ content_fetch_on_complete (GObject        *object,
                                        object_input, length,
                                        cancellable,
                                        content_fetch_on_write_complete, fetch_data);
+    }
 
  out:
   pull_data->n_outstanding_content_fetches--;
@@ -1589,7 +1613,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   OstreeRepoPullFlags flags = 0;
   const char *dir_to_pull = NULL;
   char **refs_to_fetch = NULL;
-  gboolean is_mirror;
   GSource *update_timeout = NULL;
   GSource *idle_src;
 
@@ -1609,7 +1632,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   if (dir_to_pull)
     g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
 
-  is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
+  pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
 
   pull_data->async_error = error;
   pull_data->main_context = g_main_context_ref_thread_default ();
@@ -1824,7 +1847,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
 
-  if (is_mirror && !refs_to_fetch && !configured_branches)
+  if (pull_data->is_mirror && !refs_to_fetch && !configured_branches)
     {
       SoupURI *summary_uri = NULL;
       gs_unref_bytes GBytes *bytes = NULL;
@@ -2034,7 +2057,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         }
       else
         {
-          ostree_repo_transaction_set_ref (pull_data->repo, is_mirror ? NULL : pull_data->remote_name, ref, checksum);
+          ostree_repo_transaction_set_ref (pull_data->repo, pull_data->is_mirror ? NULL : pull_data->remote_name, ref, checksum);
         }
     }
 

tests/test-pull-mirror-summary.sh

@@ -38,7 +38,7 @@ ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u
 
 cd ${test_tmpdir}
 mkdir repo
-ostree --repo=repo init
+ostree --repo=repo init --mode=archive-z2
 ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
 ostree --repo=repo pull --mirror origin
 ostree --repo=repo checkout -U main main-copy