Add an internal API to stream content objects

For future delta work where we do more interesting things than just
"tar of new objects", this lays the groundwork for doing streaming
writes into content objects.

It's also more efficient, as we avoid many intermediate allocations
and virtual calls.  Just a single `g_output_stream_write_all` for the
splice case.

Conflicts:
	src/libostree/ostree-repo-private.h
	src/libostree/ostree-repo-static-delta-processing.c

src/libostree/ostree-repo-commit.c

@@ -74,7 +74,9 @@ _ostree_repo_get_tmpobject_path (OstreeRepo       *repo,
 }
 
 static GVariant *
-create_file_metadata (GFileInfo *file_info,
+create_file_metadata (guint32       uid,
+                      guint32       gid,
+                      guint32       mode,
                       GVariant     *xattrs)
 {
   GVariant *ret_metadata = NULL;
@@ -84,9 +86,9 @@ create_file_metadata (GFileInfo *file_info,
     tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
 
   ret_metadata = g_variant_new ("(uuu@a(ayay))",
-                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::uid")),
+                                GUINT32_TO_BE (uid),
-                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::gid")),
+                                GUINT32_TO_BE (gid),
-                                GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::mode")),
+                                GUINT32_TO_BE (mode),
                                 xattrs ? xattrs : tmp_xattrs);
   g_variant_ref_sink (ret_metadata);
 
@@ -95,14 +97,16 @@ create_file_metadata (GFileInfo *file_info,
 
 static gboolean
 write_file_metadata_to_xattr (int fd,
-                              GFileInfo    *file_info,
+                              guint32       uid,
+                              guint32       gid,
+                              guint32       mode,
                               GVariant     *xattrs,
                               GError       **error)
 {
   gs_unref_variant GVariant *filemeta = NULL;
   int res;
 
-  filemeta = create_file_metadata (file_info, xattrs);
+  filemeta = create_file_metadata (uid, gid, mode, xattrs);
 
   do
     res = fsetxattr (fd, "user.ostreemeta",
@@ -125,12 +129,13 @@ commit_loose_object_trusted (OstreeRepo        *self,
                              const char        *checksum,
                              OstreeObjectType   objtype,
                              const char        *loose_path,
-                             GFile             *temp_file,
                              const char        *temp_filename,
                              gboolean           object_is_symlink,
-                             GFileInfo         *file_info,
+                             guint32            uid,
+                             guint32            gid,
+                             guint32            mode,
                              GVariant          *xattrs,
-                             GOutputStream     *temp_out,
+                             int                fd,
                              GCancellable      *cancellable,
                              GError           **error)
 {
@@ -162,8 +167,7 @@ commit_loose_object_trusted (OstreeRepo        *self,
        * as regular files.
        */
       if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename,
-                                g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
+                                uid, gid,
-                                g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
                                 AT_SYMLINK_NOFOLLOW) == -1))
         {
           gs_set_error_from_errno (error, errno);
@@ -179,22 +183,13 @@ commit_loose_object_trusted (OstreeRepo        *self,
     }
   else
     {
-      int fd;
       int res;
       struct timespec times[2];
 
-      g_assert (temp_out != NULL);
-
-      fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
-
       if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE)
         {
-          g_assert (file_info != NULL);
-
           do
-            res = fchown (fd,
+            res = fchown (fd, uid, gid);
-                          g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
-                          g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
           while (G_UNLIKELY (res == -1 && errno == EINTR));
           if (G_UNLIKELY (res == -1))
             {
@@ -203,7 +198,7 @@ commit_loose_object_trusted (OstreeRepo        *self,
             }
 
           do
-            res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
+            res = fchmod (fd, mode);
           while (G_UNLIKELY (res == -1 && errno == EINTR));
           if (G_UNLIKELY (res == -1))
             {
@@ -220,9 +215,7 @@ commit_loose_object_trusted (OstreeRepo        *self,
 
       if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER)
         {
-          g_assert (file_info != NULL);
+          if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error))
-
-          if (!write_file_metadata_to_xattr (fd, file_info, xattrs, error))
             goto out;
 
           if (!object_is_symlink)
@@ -232,7 +225,7 @@ commit_loose_object_trusted (OstreeRepo        *self,
                  checkout. To make this work we apply all user bits and the read bits for
                  group/other */
               do
-                res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode") | 0744);
+                res = fchmod (fd, mode | 0744);
               while (G_UNLIKELY (res == -1 && errno == EINTR));
               if (G_UNLIKELY (res == -1))
                 {
@@ -274,9 +267,6 @@ commit_loose_object_trusted (OstreeRepo        *self,
               goto out;
             }
         }
-          
-      if (!g_output_stream_close (temp_out, cancellable, error))
-        goto out;
     }
   
   if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path,
@@ -467,6 +457,85 @@ fallocate_stream (GFileDescriptorBased      *stream,
   return ret;
 }
 
+gboolean
+_ostree_repo_open_trusted_content_bare (OstreeRepo          *self,
+                                        const char          *checksum,
+                                        guint64              content_len,
+                                        OstreeRepoTrustedContentBareCommit *out_state,
+                                        GOutputStream      **out_stream,
+                                        gboolean            *out_have_object,
+                                        GCancellable        *cancellable,
+                                        GError             **error)
+{
+  gboolean ret = FALSE;
+  gs_free char *temp_filename = NULL;
+  gs_unref_object GOutputStream *ret_stream = NULL;
+  gboolean have_obj;
+  char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
+
+  if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE,
+                                      &have_obj, loose_objpath,
+                                      NULL,
+                                      cancellable, error))
+    goto out;
+
+  if (!have_obj)
+    {
+      if (!gs_file_open_in_tmpdir_at (self->tmp_dir_fd, 0644, &temp_filename, &ret_stream,
+                                      cancellable, error))
+        goto out;
+      
+      if (!fallocate_stream ((GFileDescriptorBased*)ret_stream, content_len,
+                             cancellable, error))
+        goto out;
+    }
+
+  ret = TRUE;
+  if (!have_obj)
+    {
+      out_state->temp_filename = temp_filename;
+      temp_filename = NULL;
+      out_state->fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)ret_stream);
+      gs_transfer_out_value (out_stream, &ret_stream);
+    }
+  *out_have_object = have_obj;
+ out:
+  return ret;
+}
+
+gboolean
+_ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
+                                          const char          *checksum,
+                                          OstreeRepoTrustedContentBareCommit *state,
+                                          guint32              uid,
+                                          guint32              gid,
+                                          guint32              mode,
+                                          GVariant            *xattrs,
+                                          GCancellable        *cancellable,
+                                          GError             **error)
+{
+  gboolean ret = FALSE;
+  char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
+
+  if (state->fd != -1)
+    {
+      _ostree_loose_path (loose_objpath, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
+      
+      if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE,
+                                        loose_objpath,
+                                        state->temp_filename,
+                                        FALSE, uid, gid, mode,
+                                        xattrs, state->fd,
+                                        cancellable, error))
+        goto out;
+    }
+
+  ret = TRUE;
+ out:
+  g_free (state->temp_filename);
+  return ret;
+}
+
 static gboolean
 write_object (OstreeRepo         *self,
               OstreeObjectType    objtype,
@@ -482,7 +551,6 @@ write_object (OstreeRepo         *self,
   gboolean do_commit;
   OstreeRepoMode repo_mode;
   gs_free char *temp_filename = NULL;
-  gs_unref_object GFile *temp_file = NULL;
   gs_unref_object GFile *stored_path = NULL;
   gs_free guchar *ret_csum = NULL;
   gs_unref_object OstreeChecksumInputStream *checksum_input = NULL;
@@ -583,7 +651,6 @@ write_object (OstreeRepo         *self,
                                  cancellable, error))
             goto out;
 
-          temp_file = g_file_get_child (self->tmp_dir, temp_filename);
           if (g_output_stream_splice (temp_out, file_input, 0,
                                       cancellable, error) < 0)
             goto out;
@@ -595,7 +662,6 @@ write_object (OstreeRepo         *self,
                                                   &temp_filename,
                                                   cancellable, error))
             goto out;
-          temp_file = g_file_get_child (self->tmp_dir, temp_filename);
         }
       else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2)
         {
@@ -610,7 +676,6 @@ write_object (OstreeRepo         *self,
                                           &temp_filename, &temp_out,
                                           cancellable, error))
             goto out;
-          temp_file = g_file_get_child (self->tmp_dir, temp_filename);
           temp_file_is_regular = TRUE;
 
           file_meta = _ostree_zlib_file_header_new (file_info, xattrs);
@@ -645,7 +710,6 @@ write_object (OstreeRepo         *self,
                              cancellable, error))
         goto out;
 
-      temp_file = g_file_get_child (self->tmp_dir, temp_filename);
       if (g_output_stream_splice (temp_out, checksum_input ? (GInputStream*)checksum_input : input,
                                   0,
                                   cancellable, error) < 0)
@@ -676,16 +740,17 @@ write_object (OstreeRepo         *self,
 
   g_assert (actual_checksum != NULL); /* Pacify static analysis */
           
-  if (indexable)
+  if (indexable && temp_file_is_regular)
     {
-      gsize archived_size;
+      struct stat stbuf;
-      gs_unref_object GFileInfo *compressed_info =
+
-        g_file_query_info (temp_file, G_FILE_ATTRIBUTE_STANDARD_SIZE, 0,
+      if (fstatat (self->tmp_dir_fd, temp_filename, &stbuf, AT_SYMLINK_NOFOLLOW) == -1)
-                           cancellable, error);
+        {
-      if (!compressed_info)
+          gs_set_error_from_errno (error, errno);
-        goto out;
+          goto out;
-      archived_size = g_file_info_get_size (compressed_info);
+        }
-      repo_store_size_entry (self, actual_checksum, unpacked_size, archived_size);
+
+      repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size);
     }
 
   if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype,
@@ -697,11 +762,27 @@ write_object (OstreeRepo         *self,
 
   if (do_commit)
     {
-      if (!commit_loose_object_trusted (self, actual_checksum,
+      guint32 uid, gid, mode;
-                                        objtype, loose_objpath,
+      int fd = -1;
-                                        temp_file, temp_filename,
+
-                                        object_is_symlink, file_info,
+      if (file_info)
-                                        xattrs, temp_out,
+        {
+          uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+          gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+          mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+        }
+      else
+        uid = gid = mode = 0;
+
+      if (temp_out)
+        fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
+      
+      if (!commit_loose_object_trusted (self, actual_checksum, objtype,
+                                        loose_objpath,
+                                        temp_filename,
+                                        object_is_symlink,
+                                        uid, gid, mode,
+                                        xattrs, fd,
                                         cancellable, error))
         goto out;
 
@@ -720,7 +801,6 @@ write_object (OstreeRepo         *self,
         }
 
       g_clear_pointer (&temp_filename, g_free);
-      g_clear_object (&temp_file);
     }
 
   g_mutex_lock (&self->txn_stats_lock);

src/libostree/ostree-repo-private.h

@@ -194,4 +194,30 @@ _ostree_repo_gpg_verify_file_with_metadata (OstreeRepo          *self,
                                             GCancellable        *cancellable,
                                             GError             **error);
 
+typedef struct {
+  int fd;
+  char *temp_filename;
+} OstreeRepoTrustedContentBareCommit;
+
+gboolean
+_ostree_repo_open_trusted_content_bare (OstreeRepo          *self,
+                                        const char          *checksum,
+                                        guint64              content_len,
+                                        OstreeRepoTrustedContentBareCommit *out_state,
+                                        GOutputStream      **out_stream,
+                                        gboolean            *out_have_object,
+                                        GCancellable        *cancellable,
+                                        GError             **error);
+
+gboolean
+_ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
+                                          const char          *checksum,
+                                          OstreeRepoTrustedContentBareCommit *state,
+                                          guint32              uid,
+                                          guint32              gid,
+                                          guint32              mode,
+                                          GVariant            *xattrs,
+                                          GCancellable        *cancellable,
+                                          GError             **error);
+
 G_END_DECLS