core: Add a function creating an archive-z2 content stream

It is quite similar to the already existing
ostree_raw_file_to_content_stream function, so I factored the common
part to a separate function. The difference is that we cannot report
the size of the resulting stream.

Can be useful for serving a "bare" repository as a faked "archive-z2"
repository.

Closes: #308
Approved by: cgwalters

apidoc/ostree-sections.txt

@@ -114,6 +114,7 @@ ostree_object_from_string
 ostree_content_stream_parse
 ostree_content_file_parse
 ostree_content_file_parse_at
+ostree_raw_file_to_archive_z2_stream
 ostree_raw_file_to_content_stream
 ostree_checksum_file_from_input
 ostree_checksum_file

src/libostree/libostree.sym

@@ -341,5 +341,5 @@ global:
 LIBOSTREE_2016.6 {
 global:
         ostree_repo_remote_fetch_summary_with_options;
-
+        ostree_raw_file_to_archive_z2_stream;
 } LIBOSTREE_2016.5;

src/libostree/ostree-core.c

@@ -400,6 +400,98 @@ write_file_header_update_checksum (GOutputStream         *out,
   return ret;
 }
 
+/*
+ * header_and_input_to_stream:
+ * @file_header: A file header
+ * @input: File raw content stream
+ * @out_input: (out): Serialized object stream
+ * @out_header_size: (out): Length of the header
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Combines @file_header and @input into a single stream.
+ */
+static gboolean
+header_and_input_to_stream (GVariant           *file_header,
+                            GInputStream       *input,
+                            GInputStream      **out_input,
+                            guint64            *out_header_size,
+                            GCancellable       *cancellable,
+                            GError            **error)
+{
+  gpointer header_data;
+  gsize header_size;
+  g_autoptr(GInputStream) ret_input = NULL;
+  g_autoptr(GPtrArray) streams = NULL;
+  g_autoptr(GOutputStream) header_out_stream = NULL;
+  g_autoptr(GInputStream) header_in_stream = NULL;
+
+  header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+
+  if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL,
+                                        cancellable, error))
+    return FALSE;
+
+  if (!g_output_stream_close (header_out_stream, cancellable, error))
+    return FALSE;
+
+  header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream);
+  header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream);
+  header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free);
+
+  streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
+
+  g_ptr_array_add (streams, g_object_ref (header_in_stream));
+  if (input)
+    g_ptr_array_add (streams, g_object_ref (input));
+
+  ret_input = (GInputStream*)ostree_chain_input_stream_new (streams);
+  ot_transfer_out_value (out_input, &ret_input);
+  if (out_header_size)
+    *out_header_size = header_size;
+
+  return TRUE;
+}
+
+/**
+ * ostree_raw_file_to_archive_z2_stream:
+ * @input: File raw content stream
+ * @file_info: A file info
+ * @xattrs: (allow-none): Optional extended attributes
+ * @out_input: (out): Serialized object stream
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Convert from a "bare" file representation into an
+ * OSTREE_OBJECT_TYPE_FILE stream suitable for ostree pull.
+ */
+gboolean
+ostree_raw_file_to_archive_z2_stream (GInputStream       *input,
+                                      GFileInfo          *file_info,
+                                      GVariant           *xattrs,
+                                      GInputStream      **out_input,
+                                      GCancellable       *cancellable,
+                                      GError            **error)
+{
+  g_autoptr(GVariant) file_header = NULL;
+  g_autoptr(GInputStream) zlib_input = NULL;
+
+  file_header = _ostree_zlib_file_header_new (file_info, xattrs);
+  if (input != NULL)
+    {
+      g_autoptr(GConverter) zlib_compressor = NULL;
+
+      zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9));
+      zlib_input = g_converter_input_stream_new (input, zlib_compressor);
+    }
+  return header_and_input_to_stream (file_header,
+                                     zlib_input,
+                                     out_input,
+                                     NULL,
+                                     cancellable,
+                                     error);
+}
+
 /**
  * ostree_raw_file_to_content_stream:
  * @input: File raw content stream
@@ -423,44 +515,20 @@ ostree_raw_file_to_content_stream (GInputStream       *input,
                                    GCancellable       *cancellable,
                                    GError            **error)
 {
-  gboolean ret = FALSE;
-  gpointer header_data;
-  gsize header_size;
-  g_autoptr(GInputStream) ret_input = NULL;
   g_autoptr(GVariant) file_header = NULL;
-  g_autoptr(GPtrArray) streams = NULL;
+  guint64 header_size;
-  g_autoptr(GOutputStream) header_out_stream = NULL;
-  g_autoptr(GInputStream) header_in_stream = NULL;
 
   file_header = file_header_new (file_info, xattrs);
-
+  if (!header_and_input_to_stream (file_header,
-  header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+                                   input,
-
+                                   out_input,
-  if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL,
+                                   &header_size,
-                                        cancellable, error))
+                                   cancellable,
-    goto out;
+                                   error))
-
+    return FALSE;
-  if (!g_output_stream_close (header_out_stream, cancellable, error))
-    goto out;
-
-  header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream);
-  header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream);
-  header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free);
-
-  streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
-
-  g_ptr_array_add (streams, g_object_ref (header_in_stream));
-  if (input)
-    g_ptr_array_add (streams, g_object_ref (input));
-  
-  ret_input = (GInputStream*)ostree_chain_input_stream_new (streams);
-
-  ret = TRUE;
-  ot_transfer_out_value (out_input, &ret_input);
   if (out_length)
     *out_length = header_size + g_file_info_get_size (file_info);
- out:
+  return TRUE;
-  return ret;
 }
 
 /**

src/libostree/ostree-core.h

@@ -274,6 +274,15 @@ gboolean ostree_content_file_parse_at (gboolean                compressed,
                                        GCancellable           *cancellable,
                                        GError                **error);
 
+_OSTREE_PUBLIC
+gboolean
+ostree_raw_file_to_archive_z2_stream (GInputStream       *input,
+                                      GFileInfo          *file_info,
+                                      GVariant           *xattrs,
+                                      GInputStream      **out_input,
+                                      GCancellable       *cancellable,
+                                      GError            **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_raw_file_to_content_stream (GInputStream       *input,
                                             GFileInfo          *file_info,

tests/test-basic-c.c

@@ -34,6 +34,143 @@ test_repo_is_not_system (gconstpointer data)
   g_assert (!ostree_repo_is_system (repo));
 }
 
+static GBytes *
+input_stream_to_bytes (GInputStream *input)
+{
+  g_autoptr(GOutputStream) mem_out_stream = NULL;
+  g_autoptr(GError) error = NULL;
+
+  if (input == NULL)
+    return g_bytes_new (NULL, 0);
+
+  mem_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+  g_output_stream_splice (mem_out_stream,
+                          input,
+                          G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+                          NULL,
+                          &error);
+  g_assert_no_error (error);
+
+  return g_memory_output_stream_steal_as_bytes (G_MEMORY_OUTPUT_STREAM (mem_out_stream));
+}
+
+static void
+test_raw_file_to_archive_z2_stream (gconstpointer data)
+{
+  OstreeRepo *repo = OSTREE_REPO (data);
+  g_autofree gchar *commit_checksum = NULL;
+  g_autoptr(GHashTable) reachable = NULL;
+  g_autoptr(GError) error = NULL;
+  /* branch name of the test repository, see setup_test_repository in libtest.sh */
+  const gchar *rev = "test2";
+  GHashTableIter iter;
+  GVariant *serialized_object;
+  guint checks = 0;
+
+  ostree_repo_resolve_rev (repo,
+                           rev,
+                           FALSE,
+                           &commit_checksum,
+                           &error);
+  g_assert_no_error (error);
+  reachable = ostree_repo_traverse_new_reachable ();
+  ostree_repo_traverse_commit (repo,
+                               commit_checksum,
+                               -1,
+                               &reachable,
+                               NULL,
+                               &error);
+  g_assert_no_error (error);
+  g_hash_table_iter_init (&iter, reachable);
+  while (g_hash_table_iter_next (&iter, (gpointer*)&serialized_object, NULL))
+    {
+      const gchar *object_checksum;
+      OstreeObjectType object_type;
+      g_autoptr(GInputStream) input = NULL;
+      g_autoptr(GFileInfo) info = NULL;
+      g_autoptr(GVariant) xattrs = NULL;
+      g_autoptr(GBytes) input_bytes = NULL;
+      g_autoptr(GInputStream) mem_input = NULL;
+      g_autoptr(GInputStream) zlib_stream = NULL;
+      g_autoptr(GBytes) zlib_bytes = NULL;
+      g_autoptr(GInputStream) mem_zlib = NULL;
+      g_autoptr(GInputStream) input2 = NULL;
+      g_autoptr(GFileInfo) info2 = NULL;
+      g_autoptr(GVariant) xattrs2 = NULL;
+      g_autoptr(GBytes) input2_bytes = NULL;
+
+      ostree_object_name_deserialize (serialized_object, &object_checksum, &object_type);
+      if (object_type != OSTREE_OBJECT_TYPE_FILE)
+        continue;
+
+      ostree_repo_load_file (repo,
+                             object_checksum,
+                             &input,
+                             &info,
+                             &xattrs,
+                             NULL,
+                             &error);
+      g_assert_no_error (error);
+
+      input_bytes = input_stream_to_bytes (input);
+      /* This is to simulate NULL input received from
+       * ostree_repo_load_file. Instead of creating the mem_input
+       * variable, I could also rewind the input stream and pass it to
+       * the function below, but this would assume that the input
+       * stream implements either the GSeekable or
+       * GFileDescriptorBased interface. */
+      if (input != NULL)
+        mem_input = g_memory_input_stream_new_from_bytes (input_bytes);
+      ostree_raw_file_to_archive_z2_stream (mem_input,
+                                            info,
+                                            xattrs,
+                                            &zlib_stream,
+                                            NULL,
+                                            &error);
+      g_assert_no_error (error);
+
+      zlib_bytes = input_stream_to_bytes (zlib_stream);
+      mem_zlib = g_memory_input_stream_new_from_bytes (zlib_bytes);
+      ostree_content_stream_parse (FALSE,
+                                   mem_zlib,
+                                   g_bytes_get_size (zlib_bytes),
+                                   FALSE,
+                                   &input2,
+                                   &info2,
+                                   &xattrs2,
+                                   NULL,
+                                   &error);
+      g_assert_error (error, G_IO_ERROR, G_IO_ERROR_FAILED);
+      g_clear_error (&error);
+
+      g_seekable_seek (G_SEEKABLE (mem_zlib),
+                       0,
+                       G_SEEK_SET,
+                       NULL,
+                       &error);
+      g_assert_no_error (error);
+
+      ostree_content_stream_parse (TRUE,
+                                   mem_zlib,
+                                   g_bytes_get_size (zlib_bytes),
+                                   FALSE,
+                                   &input2,
+                                   &info2,
+                                   &xattrs2,
+                                   NULL,
+                                   &error);
+      g_assert_no_error (error);
+
+      input2_bytes = input_stream_to_bytes (input2);
+      g_assert_true (g_bytes_equal (input_bytes, input2_bytes));
+      g_assert_true (g_variant_equal (xattrs, xattrs2));
+      /* TODO: Not sure how to compare fileinfos */
+      ++checks;
+    }
+  /* to make sure we really tested the function */
+  g_assert_cmpint (checks, >, 0);
+}
+
 int main (int argc, char **argv)
 {
   g_autoptr(GError) error = NULL;
@@ -46,6 +183,7 @@ int main (int argc, char **argv)
     goto out;
   
   g_test_add_data_func ("/repo-not-system", repo, test_repo_is_not_system);
+  g_test_add_data_func ("/raw-file-to-archive-z2-stream", repo, test_raw_file_to_archive_z2_stream);
 
   return g_test_run();
  out: