core: Add detached metadata, readd metadata to commits

Previously I thought we'd have to ditch the current commit
format to avoid a{sv} due to

See https://bugzilla.gnome.org/show_bug.cgi?id=673012

But I realized that we don't really have to care about
unpacking/repacking commit objects, so let's just re-expose the
existing metadata a{sv} in commits in the API.

Also, add support for "detached" metadata that can be updated at any
time post-commit.  This is specifically designed for GPG signatures.

https://bugzilla.gnome.org/show_bug.cgi?id=707379

src/libostree/ostree-repo-commit.c

@@ -1171,6 +1171,7 @@ create_empty_gvariant_dict (void)
  * @parent: (allow-none): ASCII SHA256 checksum for parent, or %NULL for none
  * @subject: Subject
  * @body: (allow-none): Body
+ * @metadata: (allow-none): GVariant of type a{sv}, or %NULL for none
  * @root_contents_checksum: ASCII SHA256 checksum for %OSTREE_OBJECT_TYPE_DIR_TREE
  * @root_metadata_checksum: ASCII SHA256 checksum for %OSTREE_OBJECT_TYPE_DIR_META
  * @out_commit: (out): Resulting ASCII SHA256 checksum for commit
@@ -1185,6 +1186,7 @@ ostree_repo_write_commit (OstreeRepo    *self,
                           const char    *parent,
                           const char    *subject,
                           const char    *body,
+                          GVariant      *metadata,
                           const char    *root_contents_checksum,
                           const char    *root_metadata_checksum,
                           char         **out_commit,
@@ -1203,7 +1205,7 @@ ostree_repo_write_commit (OstreeRepo    *self,
 
   now = g_date_time_new_now_utc ();
   commit = g_variant_new ("(@a{sv}@ay@a(say)sst@ay@ay)",
-                          create_empty_gvariant_dict (),
+                          metadata ? metadata : create_empty_gvariant_dict (),
                           parent ? ostree_checksum_to_bytes_v (parent) : ot_gvariant_new_bytearray (NULL, 0),
                           g_variant_new_array (G_VARIANT_TYPE ("(say)"), NULL, 0),
                           subject, body ? body : "",
@@ -1226,6 +1228,94 @@ ostree_repo_write_commit (OstreeRepo    *self,
   return ret;
 }
 
+GFile *
+_ostree_repo_get_commit_metadata_loose_path (OstreeRepo        *self,
+                                             const char        *checksum)
+{
+  gs_free char *commit_path = ostree_get_relative_object_path (checksum, OSTREE_OBJECT_TYPE_COMMIT, FALSE);
+  return ot_gfile_resolve_path_printf (self->repodir, "%smeta", commit_path);
+}
+
+/**
+ * ostree_repo_read_commit_detached_metadata:
+ * @self: Repo
+ * @checksum: ASCII SHA256 commit checksum
+ * @out_metadata: (out) (transfer full): Metadata associated with commit in with format "a{sv}", or %NULL if none exists
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * OSTree commits can have arbitrary metadata associated; this
+ * function retrieves them.  If none exists, @out_metadata will be set
+ * to %NULL.
+ */
+gboolean
+ostree_repo_read_commit_detached_metadata (OstreeRepo      *self,
+                                           const char      *checksum,
+                                           GVariant       **out_metadata,
+                                           GCancellable    *cancellable,
+                                           GError         **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_object GFile *metadata_path =
+    _ostree_repo_get_commit_metadata_loose_path (self, checksum);
+  gs_unref_variant GVariant *ret_metadata = NULL;
+  GError *temp_error = NULL;
+  
+  if (!ot_util_variant_map (metadata_path, G_VARIANT_TYPE ("a{sv}"),
+                            TRUE, &ret_metadata, &temp_error))
+    {
+      if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+        {
+          g_clear_error (&temp_error);
+        }
+      else
+        {
+          g_propagate_error (error, temp_error);
+          goto out;
+        }
+    }
+
+  ret = TRUE;
+  ot_transfer_out_value (out_metadata, &ret_metadata);
+ out:
+  return ret;
+}
+
+/**
+ * ostree_repo_write_commit_detached_metadata:
+ * @self: Repo
+ * @checksum: ASCII SHA256 commit checksum
+ * @metadata: (allow-none): Metadata to associate with commit in with format "a{sv}", or %NULL to delete
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Replace any existing metadata associated with commit referred to by
+ * @checksum with @metadata.  If @metadata is %NULL, then existing
+ * data will be deleted.
+ */
+gboolean
+ostree_repo_write_commit_detached_metadata (OstreeRepo      *self,
+                                            const char      *checksum,
+                                            GVariant        *metadata,
+                                            GCancellable    *cancellable,
+                                            GError         **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_object GFile *metadata_path =
+    _ostree_repo_get_commit_metadata_loose_path (self, checksum);
+
+  if (!g_file_replace_contents (metadata_path,
+                                g_variant_get_data (metadata),
+                                g_variant_get_size (metadata),
+                                NULL, FALSE, 0, NULL,
+                                cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
 static GVariant *
 create_tree_variant_from_hashes (GHashTable            *file_checksums,
                                  GHashTable            *dir_contents_checksums,

src/libostree/ostree-repo-private.h

@@ -74,6 +74,10 @@ _ostree_repo_find_object (OstreeRepo           *self,
                           GCancellable         *cancellable,
                           GError             **error);
 
+GFile *
+_ostree_repo_get_commit_metadata_loose_path (OstreeRepo        *self,
+                                             const char        *checksum);
+
 gboolean
 _ostree_repo_has_loose_object (OstreeRepo           *self,
                                const char           *checksum,

src/libostree/ostree-repo-prune.c

@@ -64,6 +64,13 @@ maybe_prune_loose_object (OtPruneData        *data,
 
           if (info)
             {
+              if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+                {
+                  gs_unref_object GFile *detached_metadata =
+                    _ostree_repo_get_commit_metadata_loose_path (data->repo, checksum);
+                  if (!ot_gfile_ensure_unlinked (detached_metadata, cancellable, error))
+                    goto out;
+                }
               if (!gs_file_unlink (objf, cancellable, error))
                 goto out;
               data->freed_bytes += g_file_info_get_size (info);

src/libostree/ostree-repo.h

@@ -318,12 +318,25 @@ gboolean      ostree_repo_write_commit (OstreeRepo   *self,
                                         const char   *parent,
                                         const char   *subject,
                                         const char   *body,
+                                        GVariant     *metadata,
                                         const char   *root_contents_checksum,
                                         const char   *root_metadata_checksum,
                                         char        **out_commit,
                                         GCancellable *cancellable,
                                         GError      **error);
 
+gboolean      ostree_repo_read_commit_detached_metadata (OstreeRepo      *self,
+                                                         const char      *checksum,
+                                                         GVariant       **out_metadata,
+                                                         GCancellable    *cancellable,
+                                                         GError         **error);
+
+gboolean      ostree_repo_write_commit_detached_metadata (OstreeRepo      *self,
+                                                          const char      *checksum,
+                                                          GVariant        *metadata,
+                                                          GCancellable    *cancellable,
+                                                          GError         **error);
+
 /**
  * OstreeRepoCheckoutMode:
  * @OSTREE_REPO_CHECKOUT_MODE_NONE: No special options

src/ostree/ot-builtin-commit.c

@@ -31,6 +31,8 @@ static char *opt_subject;
 static char *opt_body;
 static char *opt_branch;
 static char *opt_statoverride_file;
+static char **opt_metadata_strings;
+static char **opt_detached_metadata_strings;
 static gboolean opt_link_checkout_speedup;
 static gboolean opt_skip_if_unchanged;
 static gboolean opt_tar_autocreate_parents;
@@ -45,6 +47,8 @@ static GOptionEntry options[] = {
   { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "body" },
   { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "branch" },
   { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "NAME" },
+  { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Append given key and value (in string format) to metadata", "KEY=VALUE" },
+  { "add-detached-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_detached_metadata_strings, "Append given key and value (in string format) to detached metadata", "KEY=VALUE" },
   { "owner-uid", 0, 0, G_OPTION_ARG_INT, &opt_owner_uid, "Set file ownership user id", "UID" },
   { "owner-gid", 0, 0, G_OPTION_ARG_INT, &opt_owner_gid, "Set file ownership group id", "GID" },
   { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Do not import extended attributes", NULL },
@@ -218,6 +222,45 @@ out:
   return ret;
 }
 
+static gboolean
+parse_keyvalue_strings (char             **strings,
+                        GVariant         **out_metadata,
+                        GError           **error)
+{
+  gboolean ret = FALSE;
+  char **iter;
+  gs_unref_variant_builder GVariantBuilder *builder = NULL;
+
+  builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+
+  for (iter = strings; *iter; iter++)
+    {
+      const char *s;
+      const char *eq;
+      gs_free char *key = NULL;
+
+      s = *iter;
+
+      eq = strchr (s, '=');
+      if (!eq)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Missing '=' in KEY=VALUE metadata '%s'", s);
+          goto out;
+        }
+          
+      key = g_strndup (s, eq - s);
+      g_variant_builder_add (builder, "{sv}", key,
+                             g_variant_new_string (eq + 1));
+    }
+
+  ret = TRUE;
+  *out_metadata = g_variant_builder_end (builder);
+  g_variant_ref_sink (*out_metadata);
+ out:
+  return ret;
+}
+
 gboolean
 ostree_builtin_commit (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error)
 {
@@ -228,6 +271,8 @@ ostree_builtin_commit (int argc, char **argv, OstreeRepo *repo, GCancellable *ca
   gs_free char *parent = NULL;
   gs_free char *commit_checksum = NULL;
   gs_free char *contents_checksum = NULL;
+  gs_unref_variant GVariant *metadata = NULL;
+  gs_unref_variant GVariant *detached_metadata = NULL;
   gs_unref_object OstreeMutableTree *mtree = NULL;
   gs_free char *tree_type = NULL;
   gs_unref_hashtable GHashTable *mode_adds = NULL;
@@ -246,6 +291,19 @@ ostree_builtin_commit (int argc, char **argv, OstreeRepo *repo, GCancellable *ca
         goto out;
     }
 
+  if (opt_metadata_strings)
+    {
+      if (!parse_keyvalue_strings (opt_metadata_strings,
+                                   &metadata, error))
+        goto out;
+    }
+  if (opt_detached_metadata_strings)
+    {
+      if (!parse_keyvalue_strings (opt_detached_metadata_strings,
+                                   &detached_metadata, error))
+        goto out;
+    }
+      
   if (!opt_branch)
     {
       g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -405,10 +463,18 @@ ostree_builtin_commit (int argc, char **argv, OstreeRepo *repo, GCancellable *ca
         }
 
       if (!ostree_repo_write_commit (repo, parent, opt_subject, opt_body,
-                                     contents_checksum, root_metadata,
+                                     metadata, contents_checksum, root_metadata,
                                      &commit_checksum, cancellable, error))
         goto out;
 
+      if (detached_metadata)
+        {
+          if (!ostree_repo_write_commit_detached_metadata (repo, commit_checksum,
+                                                           detached_metadata,
+                                                           cancellable, error))
+            goto out;
+        }
+
       ostree_repo_transaction_set_ref (repo, NULL, opt_branch, commit_checksum);
 
       if (!ostree_repo_commit_transaction (repo, &stats, cancellable, error))

src/ostree/ot-builtin-show.c

@@ -30,12 +30,14 @@
 static gboolean opt_print_related;
 static char* opt_print_variant_type;
 static char* opt_print_metadata_key;
+static char* opt_print_detached_metadata_key;
 static gboolean opt_raw;
 
 static GOptionEntry options[] = {
   { "print-related", 0, 0, G_OPTION_ARG_NONE, &opt_print_related, "If given, show the \"related\" commits", NULL },
   { "print-variant-type", 0, 0, G_OPTION_ARG_STRING, &opt_print_variant_type, "If given, argument should be a filename and it will be interpreted as this type", NULL },
   { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key KEY for given commit", "KEY" },
+  { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key KEY for given commit", "KEY" },
   { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" },
   { NULL }
 };
@@ -98,22 +100,31 @@ do_print_related (OstreeRepo  *repo,
 }
 
 static gboolean
-do_print_metadata_key (OstreeRepo  *repo,
+do_print_metadata_key (OstreeRepo     *repo,
-                       const char *resolved_rev,
+                       const char     *resolved_rev,
-                       const char *key,
+                       gboolean        detached,
-                       GError **error)
+                       const char     *key,
+                       GError        **error)
 {
   gboolean ret = FALSE;
   const char *value;
   gs_unref_variant GVariant *commit = NULL;
   gs_unref_variant GVariant *metadata = NULL;
 
-  if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
+  if (!detached)
-                                 resolved_rev, &commit, error))
+    {
-    goto out;
+      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
-
+                                     resolved_rev, &commit, error))
-  /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
+        goto out;
-  metadata = g_variant_get_child_value (commit, 1);
+      /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
+      metadata = g_variant_get_child_value (commit, 0);
+    }
+  else
+    {
+      if (!ostree_repo_read_commit_detached_metadata (repo, resolved_rev, &metadata,
+                                                      NULL, error))
+        goto out;
+    }
   
   if (!g_variant_lookup (metadata, key, "&s", &value))
     goto out;
@@ -125,7 +136,6 @@ do_print_metadata_key (OstreeRepo  *repo,
   return ret;
 }
 
-
 static gboolean
 print_object (OstreeRepo          *repo,
               OstreeObjectType     objtype,
@@ -198,12 +208,14 @@ ostree_builtin_show (int argc, char **argv, OstreeRepo *repo, GCancellable *canc
     }
   rev = argv[1];
 
-  if (opt_print_metadata_key)
+  if (opt_print_metadata_key || opt_print_detached_metadata_key)
     {
+      gboolean detached = opt_print_detached_metadata_key != NULL;
+      const char *key = detached ? opt_print_detached_metadata_key : opt_print_metadata_key;
       if (!ostree_repo_resolve_rev (repo, rev, FALSE, &resolved_rev, error))
         goto out;
 
-      if (!do_print_metadata_key (repo, resolved_rev, opt_print_metadata_key, error))
+      if (!do_print_metadata_key (repo, resolved_rev, detached, key, error))
         goto out;
     }
   else if (opt_print_related)

tests/test-basic.sh

@@ -19,7 +19,7 @@
 
 set -e
 
-echo "1..40"
+echo "1..41"
 
 . $(dirname $0)/libtest.sh
 
@@ -282,3 +282,12 @@ $OSTREE checkout test2 checkout-test2
 touch checkout-test2/sometestfile
 $OSTREE commit -s sometest -b test2 checkout-test2
 echo "ok commit with directory filename"
+
+$OSTREE commit -b test2 -s "Metadata string" --add-metadata-string=FOO=BAR --add-metadata-string=KITTENS=CUTE --add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=test2
+$OSTREE show --print-metadata-key=FOO test2 > test2-meta
+assert_file_has_content test2-meta "BAR"
+$OSTREE show --print-metadata-key=KITTENS test2 > test2-meta
+assert_file_has_content test2-meta "CUTE"
+$OSTREE show --print-detached-metadata-key=SIGNATURE test2 > test2-meta
+assert_file_has_content test2-meta "HANCOCK"
+echo "ok metadata commit with strings"