core: Make compose commits contain metadata about their composition

This allows tracking of their history better.

Makefile-otutil.am

@@ -25,6 +25,8 @@ libotutil_la_SOURCES = \
 	libotutil/ot-opt-utils.h \
 	libotutil/ot-unix-utils.c \
 	libotutil/ot-unix-utils.h \
+	libotutil/ot-variant-utils.c \
+	libotutil/ot-variant-utils.h \
 	libotutil/ot-gio-utils.c \
 	libotutil/ot-gio-utils.h \
 	libotutil/ot-glib-compat.c \

libostree/ostree-core.c

@@ -335,36 +335,26 @@ ostree_parse_metadata_file (const char                  *path,
                             GVariant                   **out_variant,
                             GError                     **error)
 {
-  GMappedFile *mfile = NULL;
+  GFile *pathf = NULL;
   gboolean ret = FALSE;
   GVariant *ret_variant = NULL;
   GVariant *container = NULL;
   guint32 ret_type;
 
-  mfile = g_mapped_file_new (path, FALSE, error);
+  pathf = ot_util_new_file_for_path (path);
-  if (mfile == NULL)
+  if (!ot_util_variant_map (pathf, G_VARIANT_TYPE (OSTREE_SERIALIZED_VARIANT_FORMAT),
+                            &container, error))
+    goto out;
+
+  g_variant_ref_sink (container);
+  g_variant_get (container, "(uv)",
+                 &ret_type, &ret_variant);
+  ret_type = GUINT32_FROM_BE (ret_type);
+  if (ret_type <= 0 || ret_type > OSTREE_SERIALIZED_VARIANT_LAST)
     {
-      goto out;
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-    }
-  else
-    {
-      container = g_variant_new_from_data (G_VARIANT_TYPE (OSTREE_SERIALIZED_VARIANT_FORMAT),
-                                           g_mapped_file_get_contents (mfile),
-                                           g_mapped_file_get_length (mfile),
-                                           FALSE,
-                                           (GDestroyNotify) g_mapped_file_unref,
-                                           mfile);
-      mfile = NULL;
-      g_variant_ref_sink (container);
-      g_variant_get (container, "(uv)",
-                     &ret_type, &ret_variant);
-      ret_type = GUINT32_FROM_BE (ret_type);
-      if (ret_type <= 0 || ret_type > OSTREE_SERIALIZED_VARIANT_LAST)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                        "Corrupted metadata object '%s'; invalid type %d", path, ret_type);
-          goto out;
+      goto out;
-        }
     }
 
   ret = TRUE;
@@ -376,8 +366,7 @@ ostree_parse_metadata_file (const char                  *path,
     g_variant_unref (ret_variant);
   if (container != NULL)
     g_variant_unref (container);
-  if (mfile != NULL)
+  g_clear_object (&pathf);
-    g_mapped_file_unref (mfile);
   return ret;
 }
 

libostree/ostree-repo.c

@@ -1695,6 +1695,7 @@ ostree_repo_commit (OstreeRepo *self,
   g_return_val_if_fail (priv->inited, FALSE);
   g_return_val_if_fail (branch != NULL, FALSE);
   g_return_val_if_fail (subject != NULL, FALSE);
+  g_return_val_if_fail (metadata == NULL || g_variant_is_of_type (metadata, G_VARIANT_TYPE ("a{sv}")), FALSE);
 
   if (parent == NULL)
     parent = branch;
@@ -1778,6 +1779,7 @@ ostree_repo_commit_from_filelist_fd (OstreeRepo *self,
   g_return_val_if_fail (priv->inited, FALSE);
   g_return_val_if_fail (branch != NULL, FALSE);
   g_return_val_if_fail (subject != NULL, FALSE);
+  g_return_val_if_fail (metadata == NULL || g_variant_is_of_type (metadata, G_VARIANT_TYPE ("a{sv}")), FALSE);
 
   if (parent == NULL)
     parent = branch;

libotutil/ot-variant-utils.c

@@ -0,0 +1,113 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2011 Colin Walters <walters@verbum.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters@verbum.org>
+ */
+
+#include "config.h"
+
+#include <gio/gio.h>
+
+#include <string.h>
+
+#include "otutil.h"
+
+GHashTable *
+ot_util_variant_asv_to_hash_table (GVariant *variant)
+{
+  GHashTable *ret;
+  GVariantIter *viter;
+  char *key;
+  GVariant *value;
+  
+  ret = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref);
+  viter = g_variant_iter_new (variant);
+  while (g_variant_iter_next (viter, "{s@v}", &key, &value))
+    g_hash_table_replace (ret, key, g_variant_ref_sink (value));
+  
+  g_variant_iter_free (viter);
+  
+  return ret;
+}
+
+gboolean
+ot_util_variant_save (GFile *dest,
+                      GVariant *variant,
+                      GCancellable *cancellable,
+                      GError  **error)
+{
+  gboolean ret = FALSE;
+  GOutputStream *out = NULL;
+  gsize bytes_written;
+  
+  out = (GOutputStream*)g_file_replace (dest, NULL, 0, FALSE, cancellable, error);
+  if (!out)
+    goto out;
+
+  if (!g_output_stream_write_all (out,
+                                  g_variant_get_data (variant),
+                                  g_variant_get_size (variant),
+                                  &bytes_written,
+                                  cancellable,
+                                  error))
+    goto out;
+  if (!g_output_stream_close (out, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  g_clear_object (&out);
+  return ret;
+}
+
+gboolean
+ot_util_variant_map (GFile *src,
+                     const GVariantType *type,
+                     GVariant **out_variant,
+                     GError  **error)
+{
+  gboolean ret = FALSE;
+  GMappedFile *mfile = NULL;
+  char *path = NULL;
+  GVariant *ret_variant = NULL;
+
+  path = g_file_get_path (src);
+  mfile = g_mapped_file_new (path, FALSE, error);
+  if (!mfile)
+    goto out;
+
+  ret_variant = g_variant_new_from_data (type,
+                                         g_mapped_file_get_contents (mfile),
+                                         g_mapped_file_get_length (mfile),
+                                         FALSE,
+                                         (GDestroyNotify) g_mapped_file_unref,
+                                         mfile);
+  mfile = NULL;
+  g_variant_ref_sink (ret_variant);
+  
+  ret = TRUE;
+  *out_variant = ret_variant;
+  ret_variant = NULL;
+ out:
+  if (ret_variant)
+    g_variant_unref (ret_variant);
+  if (mfile)
+    g_mapped_file_unref (mfile);
+  g_free (path);
+  return ret;
+}

libotutil/ot-variant-utils.h

@@ -0,0 +1,43 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2011 Colin Walters <walters@verbum.org>.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters@verbum.org>
+ */
+
+#ifndef __OSTREE_VARIANT_UTILS_H__
+#define __OSTREE_VARIANT_UTILS_H__
+
+#include <gio/gio.h>
+
+G_BEGIN_DECLS
+
+GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
+
+gboolean ot_util_variant_save (GFile *dest,
+                               GVariant *variant,
+                               GCancellable *cancellable,
+                               GError  **error);
+
+gboolean ot_util_variant_map (GFile *src,
+                              const GVariantType *type,
+                              GVariant **out_variant,
+                              GError  **error);
+
+G_END_DECLS
+
+#endif

libotutil/otutil.h

@@ -21,9 +21,10 @@
 
 #ifndef __OSTREE_UTIL_H__
 
-#include <ot-unix-utils.h>
 #include <ot-gio-utils.h>
-#include <ot-opt-utils.h>
 #include <ot-glib-compat.h>
+#include <ot-opt-utils.h>
+#include <ot-unix-utils.h>
+#include <ot-variant-utils.h>
 
 #endif

ostree/ot-builtin-commit.c

@@ -204,7 +204,8 @@ ostree_builtin_commit (int argc, char **argv, const char *repo_path, GError **er
   char **iter;
   char separator;
   GVariant *metadata = NULL;
-  GMappedFile *metadata_mappedf = NULL;
+  GMappedFile *metadata_mappedf;
+  GFile *metadata_f = NULL;
 
   context = g_option_context_new ("[DIR] - Commit a new revision");
   g_option_context_add_main_entries (context, options, NULL);
@@ -245,12 +246,9 @@ ostree_builtin_commit (int argc, char **argv, const char *repo_path, GError **er
         }
       else if (metadata_bin_path)
         {
-          metadata = g_variant_new_from_data (G_VARIANT_TYPE ("a{sv}"),
+          metadata_f = ot_util_new_file_for_path (metadata_bin_path);
-                                              g_mapped_file_get_contents (metadata_mappedf),
+          if (!ot_util_variant_map (metadata_f, G_VARIANT_TYPE ("a{sv}"), &metadata, error))
-                                              g_mapped_file_get_length (metadata_mappedf),
+            goto out;
-                                              FALSE,
-                                              (GDestroyNotify) g_mapped_file_unref,
-                                              metadata_mappedf);
         }
       else
         g_assert_not_reached ();
@@ -320,7 +318,7 @@ ostree_builtin_commit (int argc, char **argv, const char *repo_path, GError **er
             }
           from_fd = temp_fd;
         }
-      if (!ostree_repo_commit_from_filelist_fd (repo, branch, parent, subject, body, NULL,
+      if (!ostree_repo_commit_from_filelist_fd (repo, branch, parent, subject, body, metadata,
                                                 dir, from_fd, separator,
                                                 &commit_checksum, error))
         {
@@ -353,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, const char *repo_path, GError **er
       if (g_thread_create (find_thread, &fdata, FALSE, error) == NULL)
         goto out;
 
-      if (!ostree_repo_commit_from_filelist_fd (repo, branch, parent, subject, body, NULL,
+      if (!ostree_repo_commit_from_filelist_fd (repo, branch, parent, subject, body, metadata,
                                                 dir, pipefd[0], separator,
                                                 &commit_checksum, error))
         goto out;

ostree/ot-builtin-compose.c

@@ -28,8 +28,10 @@
 
 #define FAST_QUERYINFO "standard::name,standard::type,standard::is-symlink,standard::symlink-target,unix::*"
 
+static char *compose_metadata_path;
+
 static GOptionEntry options[] = {
-  { NULL }
+  { "out-metadata", 0, 0, G_OPTION_ARG_FILENAME, &compose_metadata_path, "Output a file containing serialized metadata about the compose, in host endianness", "path" },
 };
 
 static void
@@ -175,6 +177,7 @@ static gboolean
 compose_branch_on_dir (OstreeRepo *repo,
                        GFile *destination,
                        const char *branch,
+                       GVariantBuilder *metadata_builder,
                        GError **error)
 {
   char *destpath = NULL;
@@ -196,11 +199,14 @@ compose_branch_on_dir (OstreeRepo *repo,
   g_print ("Checking out %s (commit %s)...\n", branch, branchrev);
   if (!ostree_repo_checkout (repo, branchrev, branchpath, error))
     goto out;
-  g_print ("...done\n", branch);
+  g_print ("...done\n");
-  g_print ("Merging over destination...\n", branch, branchrev);
+  g_print ("Merging over destination...\n");
   if (!merge_dir (destination, branchf, error))
     goto out;
 
+  if (metadata_builder)
+    g_variant_builder_add (metadata_builder, "(ss)", branch, branchrev);
+
   ret = TRUE;
  out:
   if (branchf)
@@ -222,6 +228,12 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
   OstreeCheckout *checkout = NULL;
   const char *destination;
   GFile *destf = NULL;
+  gboolean compose_metadata_builder_initialized = FALSE;
+  GVariantBuilder compose_metadata_builder;
+  gboolean commit_metadata_builder_initialized = FALSE;
+  GVariantBuilder commit_metadata_builder;
+  GVariant *commit_metadata = NULL;
+  GFile *metadata_f = NULL;
   int i;
 
   context = g_option_context_new ("DESTINATION BRANCH1 BRANCH2 ... - Merge multiple commits into a single filesystem tree");
@@ -234,10 +246,6 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
   if (!ostree_repo_check (repo, error))
     goto out;
 
-  destination = argv[1];
-  
-  destf = ot_util_new_file_for_path (destination);
-  
   if (argc < 3)
     {
       gchar *help = g_option_context_get_help (context, TRUE, NULL);
@@ -248,20 +256,52 @@ ostree_builtin_compose (int argc, char **argv, const char *repo_path, GError **e
       goto out;
     }
 
+  destination = argv[1];
+  destf = ot_util_new_file_for_path (destination);
+  
+  if (compose_metadata_path)
+    {
+      compose_metadata_builder_initialized = TRUE;
+      g_variant_builder_init (&compose_metadata_builder, G_VARIANT_TYPE ("a(ss)"));
+    }
+  
   for (i = 2; i < argc; i++)
     {
       const char *branch = argv[i];
       
-      if (!compose_branch_on_dir (repo, destf, branch, error))
+      if (!compose_branch_on_dir (repo, destf, branch, compose_metadata_builder_initialized ? &compose_metadata_builder : NULL, error))
+        goto out;
+    }
+
+  if (compose_metadata_path)
+    {
+      commit_metadata_builder_initialized = TRUE;
+      g_variant_builder_init (&commit_metadata_builder, G_VARIANT_TYPE ("a{sv}"));
+
+      g_variant_builder_add (&commit_metadata_builder, "{sv}",
+                             "ostree-compose", g_variant_builder_end (&compose_metadata_builder));
+      compose_metadata_builder_initialized = FALSE;
+
+      metadata_f = ot_util_new_file_for_path (compose_metadata_path);
+
+      commit_metadata = g_variant_builder_end (&commit_metadata_builder);
+      if (!ot_util_variant_save (metadata_f, commit_metadata, NULL, error))
         goto out;
     }
 
   ret = TRUE;
  out:
+  if (compose_metadata_builder_initialized)
+    g_variant_builder_clear (&compose_metadata_builder);
+  if (commit_metadata_builder_initialized)
+    g_variant_builder_clear (&commit_metadata_builder);
   if (context)
     g_option_context_free (context);
+  if (commit_metadata)
+    g_variant_unref (commit_metadata);
   g_clear_object (&repo);
   g_clear_object (&checkout);
   g_clear_object (&destf);
+  g_clear_object (&metadata_f);
   return ret;
 }

ostree/ot-builtin-show.c

@@ -26,10 +26,122 @@
 
 #include <glib/gi18n.h>
 
+static gboolean print_compose;
+static char* print_variant_type;
+
 static GOptionEntry options[] = {
+  { "print-compose", 0, 0, G_OPTION_ARG_NONE, &print_compose, "If given, show the branches which make up the given compose commit", NULL },
+  { "print-variant-type", 0, 0, G_OPTION_ARG_STRING, &print_variant_type, "If given, argument should be a filename and it will be interpreted as this type", NULL },
   { NULL }
 };
 
+static void
+print_variant (GVariant *variant)
+{
+  char *formatted_variant = NULL;
+
+  formatted_variant = g_variant_print (variant, TRUE);
+  g_print ("%s\n", formatted_variant);
+
+  g_free (formatted_variant);
+}
+
+static gboolean
+do_print_variant_generic (const GVariantType *type,
+                          const char *filename,
+                          GError **error)
+{
+  gboolean ret = FALSE;
+  GFile *f = NULL;
+  GVariant *variant = NULL;
+
+  f = ot_util_new_file_for_path (filename);
+
+  if (!ot_util_variant_map (f, type, &variant, error))
+    goto out;
+
+  print_variant (variant);
+
+  ret = TRUE;
+ out:
+  if (variant)
+    g_variant_unref (variant);
+  g_clear_object (&f);
+  return ret;
+}
+
+static gboolean
+show_repo_meta (OstreeRepo  *repo,
+                const char *rev,
+                const char *resolved_rev,
+                GError **error)
+{
+  OstreeSerializedVariantType type;
+  gboolean ret = FALSE;
+  GVariant *variant = NULL;
+
+  if (!ostree_repo_load_variant (repo, resolved_rev, &type, &variant, error))
+    goto out;
+  g_print ("Object: %s\nType: %d\n", resolved_rev, type);
+  print_variant (variant);
+
+  ret = TRUE;
+ out:
+  if (variant)
+    g_variant_unref (variant);
+  return ret;
+}
+
+static gboolean
+do_print_compose (OstreeRepo  *repo,
+                  const char *rev,
+                  const char *resolved_rev,
+                  GError **error)
+{
+  gboolean ret = FALSE;
+  GVariant *variant = NULL;
+  GVariant *metadata = NULL;
+  GVariant *compose_contents = NULL;
+  GVariantIter *viter = NULL;
+  GHashTable *metadata_hash = NULL;
+  const char *branch;
+  const char *branchrev;
+
+  if (!ostree_repo_load_variant_checked (repo, OSTREE_SERIALIZED_COMMIT_VARIANT,
+                                         resolved_rev, &variant, error))
+    goto out;
+      
+  /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
+  metadata = g_variant_get_child_value (variant, 1);
+  metadata_hash = ot_util_variant_asv_to_hash_table (metadata);
+  
+  compose_contents = g_hash_table_lookup (metadata_hash, "ostree-compose");
+  if (!compose_contents)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Commit %s does not have compose metadata key \"ostree-compose\"", resolved_rev);
+      goto out;
+    }
+
+  g_variant_get_child (compose_contents, 0, "a(ss)", &viter);
+  while (g_variant_iter_next (viter, "(&s&s)", &branch, &branchrev))
+    {
+      g_print ("%s %s\n", branch, branchrev);
+    }
+
+  ret = TRUE;
+ out:
+  if (variant)
+    g_variant_unref (variant);
+  if (viter)
+    g_variant_iter_free (viter);
+  if (metadata)
+    g_variant_unref (metadata);
+  if (metadata_hash)
+    g_hash_table_destroy (metadata_hash);
+  return ret;
+}
+
 gboolean
 ostree_builtin_show (int argc, char **argv, const char *repo_path, GError **error)
 {
@@ -38,9 +150,6 @@ ostree_builtin_show (int argc, char **argv, const char *repo_path, GError **erro
   OstreeRepo *repo = NULL;
   const char *rev = "master";
   char *resolved_rev = NULL;
-  OstreeSerializedVariantType type;
-  GVariant *variant = NULL;
-  char *formatted_variant = NULL;
 
   context = g_option_context_new ("- Output a metadata object");
   g_option_context_add_main_entries (context, options, NULL);
@@ -55,15 +164,27 @@ ostree_builtin_show (int argc, char **argv, const char *repo_path, GError **erro
   if (argc > 1)
     rev = argv[1];
 
-  if (!ostree_repo_resolve_rev (repo, rev, &resolved_rev, error))
+  if (print_compose)
-    goto out;
+    {
+      if (!ostree_repo_resolve_rev (repo, rev, &resolved_rev, error))
+        goto out;
 
-  if (!ostree_repo_load_variant (repo, resolved_rev, &type, &variant, error))
+      if (!do_print_compose (repo, rev, resolved_rev, error))
-    goto out;
+        goto out;
+    }
+  else if (print_variant_type)
+    {
+      if (!do_print_variant_generic (G_VARIANT_TYPE (print_variant_type), rev, error))
+        goto out;
+    }
+  else
+    {
+      if (!ostree_repo_resolve_rev (repo, rev, &resolved_rev, error))
+        goto out;
 
-  g_print ("Object: %s\nType: %d\n", resolved_rev, type);
+      if (!show_repo_meta (repo, rev, resolved_rev, error))
-  formatted_variant = g_variant_print (variant, TRUE);
+        goto out;
-  g_print ("%s\n", formatted_variant);
+    }
  
   ret = TRUE;
  out:
@@ -71,8 +192,5 @@ ostree_builtin_show (int argc, char **argv, const char *repo_path, GError **erro
   if (context)
     g_option_context_free (context);
   g_clear_object (&repo);
-  if (variant)
-    g_variant_unref (variant);
-  g_free (formatted_variant);
   return ret;
 }

tests/t0005-compose.sh

@@ -20,7 +20,7 @@
 
 set -e
 
-echo "1..3"
+echo "1..6"
 
 . libtest.sh
 
@@ -69,3 +69,17 @@ find | md5sum > ../some-compose-md5
 assert_file_has_content ../some-compose-md5 9038703e43d2ff2745fb7dd844de65c8 
 
 echo 'ok compose content'
+
+cd "${test_tmpdir}"
+rm -rf some-compose
+$OSTREE compose --out-metadata=./some-compose-metadata some-compose artifact-libfoo-runtime artifact-libfoo-devel artifact-barapp
+echo 'ok compose output metadata'
+
+cd some-compose
+$OSTREE commit --metadata-variant=${test_tmpdir}/some-compose-metadata -b some-compose -s 'Initial commit of some-compose'
+echo 'ok compose commit with metadata'
+
+$OSTREE show --print-compose some-compose > ${test_tmpdir}/some-compose-contents
+assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-runtime
+assert_file_has_content ${test_tmpdir}/some-compose-contents artifact-libfoo-devel
+echo 'ok compose verify metadata'