core: Change compose to operate purely in-memory

This is *significantly* faster than checking out each branch into the
real filesystem, then importing it again.

src/libostree/ostree-mutable-tree.c

@@ -95,6 +95,31 @@ ostree_mutable_tree_set_contents_checksum (OstreeMutableTree *self,
 const char *
 ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self)
 {
+  GHashTableIter iter;
+  gpointer key, value;
+
+  if (!self->contents_checksum)
+    return NULL;
+
+  /* Ensure the cache is valid; this implementation is a bit
+   * lame in that we walk the whole tree every time this
+   * getter is called; a better approach would be to invalidate
+   * all of the parents whenever a child is modified.
+   *
+   * However, we only call this function once right now.
+   */
+  g_hash_table_iter_init (&iter, self->subdirs);
+  while (g_hash_table_iter_next (&iter, &key, &value))
+    {
+      OstreeMutableTree *subdir = value;
+      if (!ostree_mutable_tree_get_contents_checksum (subdir))
+        {
+          g_free (self->contents_checksum);
+          self->contents_checksum = NULL;
+          return NULL;
+        }
+    }
+
   return self->contents_checksum;
 }
 
@@ -122,6 +147,7 @@ ostree_mutable_tree_replace_file (OstreeMutableTree *self,
       goto out;
     }
 
+  ostree_mutable_tree_set_contents_checksum (self, NULL);
   g_hash_table_replace (self->files,
                         g_strdup (name),
                         g_strdup (checksum));
@@ -153,6 +179,7 @@ ostree_mutable_tree_ensure_dir (OstreeMutableTree *self,
   if (!ret_dir)
     {
       ret_dir = ostree_mutable_tree_new ();
+      ostree_mutable_tree_set_contents_checksum (self, NULL);
       g_hash_table_insert (self->subdirs, g_strdup (name), g_object_ref (ret_dir));
     }
   

src/libostree/ostree-repo-file.c

@@ -280,6 +280,12 @@ ostree_repo_file_ensure_resolved (OstreeRepoFile  *self,
     return TRUE;
 }
 
+const char *
+ostree_repo_file_get_commit (OstreeRepoFile  *self)
+{
+  return ostree_repo_file_get_root (self)->commit;
+}
+
 gboolean
 ostree_repo_file_get_xattrs (OstreeRepoFile  *self,
                               GVariant       **out_xattrs,

src/libostree/ostree-repo-file.h

@@ -75,6 +75,8 @@ gboolean ostree_repo_file_is_tree (OstreeRepoFile  *self);
 
 const char * ostree_repo_file_get_checksum (OstreeRepoFile  *self);
 
+const char * ostree_repo_file_get_commit (OstreeRepoFile  *self);
+
 GFile *ostree_repo_file_nontree_get_local (OstreeRepoFile  *self);
 
 int     ostree_repo_file_tree_find_child  (OstreeRepoFile  *self,

src/libostree/ostree-repo.c

@@ -1380,6 +1380,7 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo           *self,
   GChecksum *child_file_checksum = NULL;
   GVariant *xattrs = NULL;
   GInputStream *file_input = NULL;
+  gboolean repo_dir_was_empty = FALSE;
 
   /* We can only reuse checksums directly if there's no modifier */
   if (OSTREE_IS_REPO_FILE (dir) && modifier == NULL)
@@ -1388,7 +1389,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo           *self,
   if (repo_dir)
     {
       ostree_mutable_tree_set_metadata_checksum (mtree, ostree_repo_file_get_checksum (repo_dir));
-      ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
+      repo_dir_was_empty = 
+        g_hash_table_size (ostree_mutable_tree_get_files (mtree)) == 0
+        && g_hash_table_size (ostree_mutable_tree_get_subdirs (mtree)) == 0;
     }
   else
     {
@@ -1484,6 +1487,9 @@ ostree_repo_stage_directory_to_mtree (OstreeRepo           *self,
       goto out;
     }
 
+  if (repo_dir && repo_dir_was_empty)
+    ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
+
   ret = TRUE;
  out:
   g_clear_object (&dir_enum);
@@ -1593,10 +1599,12 @@ ostree_repo_stage_mtree (OstreeRepo           *self,
   GVariant *serialized_tree = NULL;
   GHashTableIter hash_iter;
   gpointer key, value;
+  const char *existing_checksum;
 
-  if (ostree_mutable_tree_get_contents_checksum (mtree))
+  existing_checksum = ostree_mutable_tree_get_contents_checksum (mtree);
+  if (existing_checksum)
     {
-      ret_contents_checksum = g_strdup (ostree_mutable_tree_get_contents_checksum (mtree));
+      ret_contents_checksum = g_strdup (existing_checksum);
     }
   else
     {

src/ostree/ot-builtin-compose.c

@@ -27,97 +27,43 @@
 
 #include <glib/gi18n.h>
 
-static char *compose_metadata_path;
+static char *subject;
+static char *body;
+static char *branch;
 
 static GOptionEntry options[] = {
-  { "out-metadata", 0, 0, G_OPTION_ARG_FILENAME, &compose_metadata_path, "Output a file containing serialized metadata about the compose, in host endianness", "path" },
+  { "subject", 's', 0, G_OPTION_ARG_STRING, &subject, "One line subject", "subject" },
+  { "body", 'm', 0, G_OPTION_ARG_STRING, &body, "Full description", "body" },
+  { "branch", 'b', 0, G_OPTION_ARG_STRING, &branch, "Branch", "branch" },
   { NULL }
 };
 
-static void
-rm_rf (GFile *path)
-{
-  GFileInfo *finfo = NULL;
-  GFileEnumerator *path_enum = NULL;
-  guint32 type;
-  
-  finfo = g_file_query_info (path, OSTREE_GIO_FAST_QUERYINFO,
-                             G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                             NULL, NULL);
-  if (!finfo)
-    goto out;
-
-  type = g_file_info_get_attribute_uint32 (finfo, "standard::type");
-  if (type == G_FILE_TYPE_DIRECTORY)
-    {
-      path_enum = g_file_enumerate_children (path, OSTREE_GIO_FAST_QUERYINFO, 
-                                             G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                             NULL, NULL);
-      if (!path_enum)
-        goto out;
-
-      
-      g_clear_object (&finfo);
-      while ((finfo = g_file_enumerator_next_file (path_enum, NULL, NULL)) != NULL)
-        {
-          GFile *child = g_file_get_child (path, g_file_info_get_attribute_byte_string (finfo, "standard::name"));
-          rm_rf (child);
-          g_clear_object (&child);
-          g_clear_object (&finfo);
-        }
-    }
-
-  (void) unlink (ot_gfile_get_path_cached (path));
-
- out:
-  g_clear_object (&finfo);
-  g_clear_object (&path_enum);
-}
-
 static gboolean
-compose_branch_on_dir (OstreeRepo *repo,
+add_branch (OstreeRepo          *repo,
-                       GFile *destination,
+            OstreeMutableTree   *mtree,
-                       const char *branch,
+            const char          *branch,
-                       GVariantBuilder *metadata_builder,
+            GVariantBuilder     *metadata_builder,
-                       GError **error)
+            GError             **error)
 {
-  char *destpath = NULL;
-  char *branchpath = NULL;
-  GFile *branchf = NULL;
-  GFileEnumerator *enumerator = NULL;
   gboolean ret = FALSE;
-  char *branchrev = NULL;
+  GFile *branchf = NULL;
+  const char *branch_rev;
 
-  if (!ostree_repo_resolve_rev (repo, branch, FALSE, &branchrev, error))
+  if (!ostree_repo_read_commit (repo, branch, &branchf, NULL, error))
+    goto out;
+
+  branch_rev = ostree_repo_file_get_commit ((OstreeRepoFile*)branchf);
+
+  if (!ostree_repo_stage_directory_to_mtree (repo, branchf, mtree, NULL,
+                                             NULL, error))
     goto out;
   
-  destpath = g_strdup (ot_gfile_get_path_cached (destination));
-  if (g_str_has_suffix (destpath, "/"))
-    destpath[strlen (destpath) - 1] = '\0';
-  branchpath = g_strconcat (destpath, "-tmp-checkout-", branchrev, NULL);
-  branchf = ot_gfile_new_for_path (branchpath);
-
-  g_print ("Checking out %s (commit %s)...\n", branch, branchrev);
-  if (!ostree_repo_checkout (repo, OSTREE_REPO_CHECKOUT_MODE_NONE,
-                             branchrev, branchf, NULL, error))
-    goto out;
-  g_print ("...done\n");
-  g_print ("Merging over destination...\n");
-  if (!ot_gfile_merge_dirs (destination, branchf, NULL, error))
-    goto out;
-
   if (metadata_builder)
-    g_variant_builder_add (metadata_builder, "(ss)", branch, branchrev);
+    g_variant_builder_add (metadata_builder, "(ss)", branch, branch_rev);
 
   ret = TRUE;
  out:
-  if (branchf)
-    rm_rf (branchf);
-  g_free (destpath);
-  g_clear_object (&enumerator);
   g_clear_object (&branchf);
-  g_free (branchrev);
-  g_free (branchpath);
   return ret;
 }
 
@@ -128,17 +74,21 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
   gboolean ret = FALSE;
   OstreeRepo *repo = NULL;
   OstreeCheckout *checkout = NULL;
-  const char *destination;
+  char *parent = NULL;
   GFile *destf = NULL;
   gboolean compose_metadata_builder_initialized = FALSE;
   GVariantBuilder compose_metadata_builder;
   gboolean commit_metadata_builder_initialized = FALSE;
   GVariantBuilder commit_metadata_builder;
   GVariant *commit_metadata = NULL;
+  char *contents_checksum = NULL;
+  char *commit_checksum = NULL;
+  GCancellable *cancellable = NULL;
   GFile *metadata_f = NULL;
+  OstreeMutableTree *mtree = NULL;
   int i;
 
-  context = g_option_context_new ("DESTINATION BRANCH1 BRANCH2 ... - Merge multiple commits into a single filesystem tree");
+  context = g_option_context_new ("BRANCH1 BRANCH2 ... - Merge multiple commits into a single commit tree");
   g_option_context_add_main_entries (context, options, NULL);
 
   if (!g_option_context_parse (context, &argc, &argv, error))
@@ -148,50 +98,66 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
   if (!ostree_repo_check (repo, error))
     goto out;
 
-  if (argc < 3)
+  if (!branch)
     {
-      gchar *help = g_option_context_get_help (context, TRUE, NULL);
-      g_printerr ("%s\n", help);
-      g_free (help);
       g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                               "DESTINATION and at least one COMMIT must be specified");
+                           "A branch must be specified with --branch");
       goto out;
     }
 
-  destination = argv[1];
+  if (!subject)
-  destf = ot_gfile_new_for_path (destination);
-  
-  if (compose_metadata_path)
     {
-      compose_metadata_builder_initialized = TRUE;
+      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-      g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)"));
+                           "A subject must be specified with --subject");
+      goto out;
     }
+
+  compose_metadata_builder_initialized = TRUE;
+  g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)"));
+
+  if (!ostree_repo_resolve_rev (repo, branch, TRUE, &parent, error))
+    goto out;
+
+  if (!ostree_repo_prepare_transaction (repo, cancellable, error))
+    goto out;
+
+  mtree = ostree_mutable_tree_new ();
   
-  for (i = 2; i < argc; i++)
+  for (i = 1; i < argc; i++)
     {
-      const char *branch = argv[i];
+      const char *src_branch = argv[i];
       
-      if (!compose_branch_on_dir (repo, destf, branch, compose_metadata_builder_initialized ? &compose_metadata_builder : NULL, error))
+      if (!add_branch (repo, mtree, src_branch,
+                       compose_metadata_builder_initialized ? &compose_metadata_builder : NULL,
+                       error))
         goto out;
     }
 
-  if (compose_metadata_path)
+  commit_metadata_builder_initialized = TRUE;
-    {
+  g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}"));
-      commit_metadata_builder_initialized = TRUE;
-      g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}"));
 
-      g_variant_builder_add (&commit_metadata_builder, "{sv}",
+  g_variant_builder_add (&commit_metadata_builder, "{sv}",
-                             "ostree-compose", g_variant_builder_end (&compose_metadata_builder));
+                         "ostree-compose", g_variant_builder_end (&compose_metadata_builder));
-      compose_metadata_builder_initialized = FALSE;
+  commit_metadata = g_variant_builder_end (&commit_metadata_builder);
+  g_variant_ref_sink (commit_metadata);
 
-      metadata_f = ot_gfile_new_for_path (compose_metadata_path);
+  if (!ostree_repo_stage_mtree (repo, mtree, &contents_checksum, cancellable, error))
+    goto out;
 
-      commit_metadata = g_variant_builder_end (&commit_metadata_builder);
+  if (!ostree_repo_stage_commit (repo, branch, parent, subject, body, commit_metadata,
-      if (!ot_util_variant_save (metadata_f, commit_metadata, NULL, error))
+                                 contents_checksum,
-        goto out;
+                                 ostree_mutable_tree_get_metadata_checksum (mtree),
-    }
+                                 &commit_checksum, cancellable, error))
+    goto out;
+
+  if (!ostree_repo_commit_transaction (repo, cancellable, error))
+    goto out;
+
+  if (!ostree_repo_write_ref (repo, NULL, branch, commit_checksum, error))
+    goto out;
 
   ret = TRUE;
+  g_print ("%s\n", commit_checksum);
  out:
   if (compose_metadata_builder_initialized)
     g_variant_builder_clear (&compose_metadata_builder);
@@ -199,10 +165,14 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
     g_variant_builder_clear (&commit_metadata_builder);
   if (context)
     g_option_context_free (context);
+  g_free (parent);
+  g_free (contents_checksum);
+  g_free (commit_checksum);
   ot_clear_gvariant (&commit_metadata);
   g_clear_object (&repo);
   g_clear_object (&checkout);
   g_clear_object (&destf);
   g_clear_object (&metadata_f);
+  g_clear_object (&mtree);
   return ret;
 }

tests/t0004-compose.sh

@@ -19,7 +19,7 @@
 
 set -e
 
-echo "1..8"
+echo "1..5"
 
 . libtest.sh
 
@@ -58,10 +58,11 @@ $OSTREE commit -b artifact-barapp -s 'Build 42 of barapp'
 echo 'ok artifacts committed'
 
 cd "${test_tmpdir}"
-$OSTREE compose some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
+$OSTREE compose -s "compose 1" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
-echo 'ok compose command'
+echo 'ok compose'
 
-cd some-compose
+$OSTREE checkout some-compose some-compose-checkout
+cd some-compose-checkout
 assert_file_has_content ./usr/bin/bar 'another ELF file'
 assert_file_has_content ./usr/share/doc/foo.txt 'some documentation'
 find | md5sum > ../some-compose-md5
@@ -70,36 +71,15 @@ assert_file_has_content ../some-compose-md5 9038703e43d2ff2745fb7dd844de65c8
 echo 'ok compose content'
 
 cd "${test_tmpdir}"
-rm -rf some-compose
+rm -rf some-compose-checkout some-compose-metadata
-$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
-echo 'ok compose output metadata'
-
-cd some-compose
-$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Initial commit of some-compose'
-echo 'ok compose commit with metadata'
-
-$OSTREE show --print-compose some-compose > ${test_tmpdir}/some-compose-contents
-assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-runtime
-assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-devel
-echo 'ok compose verify metadata'
-
-cd "${test_tmpdir}"
-rm -rf some-compose some-compose-metadata
 cd "${test_tmpdir}"/artifact-barapp
 echo 'updated bar ELF file' > usr/bin/bar
 $OSTREE commit -b artifact-barapp -s 'Build 43 of barapp'
+$OSTREE compose -s "compose 2" -b some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
+echo 'ok compose update commit'
 
 cd "${test_tmpdir}"
-$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
-cd some-compose
-assert_file_has_content ./usr/bin/bar 'updated bar ELF file'
-
-echo 'ok updated artifact barapp'
-$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Updated some-compose'
-cd ${test_tmpdir}
-rm -rf some-compose
-
 $OSTREE checkout some-compose some-compose-checkout
 cd some-compose-checkout
 assert_file_has_content ./usr/bin/bar 'updated bar ELF file'
-echo 'ok updated compose commit'
+echo 'ok compose update contents'