Add support for more selective pruning

There are use cases for having a single repo with branches
with different lifecycles; a simple example of what I was
trying to do in CentOS Atomic Host work is have "stable"
and "devel" branches, were we want to prune devel, but
retain *all* of stable.

This patch is split into two parts - first we add a low level "delete all
objects not in this set" API, and change the current prune API
to use this.

Next, we move more logic into the "ostree prune" command. This paves the way for
demonstrating how more sophisticated algorithms/logic could be developed outside
of the ostree core.

Also, the --keep-younger-than logic already lived in the commandline, so it
makes sense to keep extending it there.

Closes: https://github.com/ostreedev/ostree/issues/604

Closes: #646
Approved by: jlebon

src/libostree/libostree.sym

@@ -371,12 +371,10 @@ global:
  *                         NOTE NOTE NOTE
  */
 
-/* Remove comment when first new symbol is added
-LIBOSTREE_2016.XX {
+LIBOSTREE_2017.1 {
 global:
-	someostree_symbol_deleteme;
+        ostree_repo_prune_from_reachable;
 } LIBOSTREE_2016.14;
-*/
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste

src/libostree/ostree-repo-prune.c

@@ -254,6 +254,61 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit,
   return ret;
 }
 
+static gboolean
+repo_prune_internal (OstreeRepo        *self,
+                     GHashTable        *objects,
+                     OstreeRepoPruneOptions *options,
+                     gint              *out_objects_total,
+                     gint              *out_objects_pruned,
+                     guint64           *out_pruned_object_size_total,
+                     GCancellable      *cancellable,
+                     GError           **error)
+{
+  gboolean ret = FALSE;
+  GHashTableIter hash_iter;
+  gpointer key, value;
+  OtPruneData data = { 0, };
+
+  data.repo = self;
+  data.reachable = g_hash_table_ref (options->reachable);
+
+  g_hash_table_iter_init (&hash_iter, objects);
+  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+    {
+      GVariant *serialized_key = key;
+      GVariant *objdata = value;
+      const char *checksum;
+      OstreeObjectType objtype;
+      gboolean is_loose;
+
+      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+      g_variant_get_child (objdata, 0, "b", &is_loose);
+
+      if (!is_loose)
+        continue;
+
+      if (!maybe_prune_loose_object (&data, options->flags, checksum, objtype,
+                                     cancellable, error))
+        goto out;
+    }
+
+  if (!ostree_repo_prune_static_deltas (self, NULL, cancellable, error))
+    goto out;
+
+  if (!_ostree_repo_prune_tmp (self, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+  *out_objects_total = (data.n_reachable_meta + data.n_unreachable_meta +
+                        data.n_reachable_content + data.n_unreachable_content);
+  *out_objects_pruned = (data.n_unreachable_meta + data.n_unreachable_content);
+  *out_pruned_object_size_total = data.freed_bytes;
+ out:
+  if (data.reachable)
+    g_hash_table_unref (data.reachable);
+  return ret;
+}
+
 /**
  * ostree_repo_prune:
  * @self: Repo
@@ -289,39 +344,42 @@ ostree_repo_prune (OstreeRepo        *self,
                    GCancellable      *cancellable,
                    GError           **error)
 {
-  gboolean ret = FALSE;
   GHashTableIter hash_iter;
   gpointer key, value;
   g_autoptr(GHashTable) objects = NULL;
   g_autoptr(GHashTable) all_refs = NULL;
-  OtPruneData data = { 0, };
+  g_autoptr(GHashTable) reachable = NULL;
   gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
 
-  data.repo = self;
-  data.reachable = ostree_repo_traverse_new_reachable ();
+  reachable = ostree_repo_traverse_new_reachable ();
+
+  /* This original prune API has fixed logic for traversing refs or all commits
+   * combined with actually deleting content. The newer backend API just does
+   * the deletion.
+   */
 
   if (refs_only)
     {
       if (!ostree_repo_list_refs (self, NULL, &all_refs,
                                   cancellable, error))
-        goto out;
-      
+        return FALSE;
+
       g_hash_table_iter_init (&hash_iter, all_refs);
-      
+
       while (g_hash_table_iter_next (&hash_iter, &key, &value))
         {
           const char *checksum = value;
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable,
+          if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
                                                   cancellable, error))
-            goto out;
+            return FALSE;
         }
     }
 
   if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,
                                  &objects, cancellable, error))
-    goto out;
+    return FALSE;
 
   if (!refs_only)
     {
@@ -338,45 +396,57 @@ ostree_repo_prune (OstreeRepo        *self,
             continue;
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable,
+          if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
                                                   cancellable, error))
-            goto out;
+            return FALSE;
         }
     }
 
-  g_hash_table_iter_init (&hash_iter, objects);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      GVariant *objdata = value;
-      const char *checksum;
-      OstreeObjectType objtype;
-      gboolean is_loose;
-      
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-      g_variant_get_child (objdata, 0, "b", &is_loose);
+  { OstreeRepoPruneOptions opts = { flags, reachable };
+    return repo_prune_internal (self, objects, &opts,
+                                out_objects_total, out_objects_pruned,
+                                out_pruned_object_size_total, cancellable, error);
+  }
+}
 
-      if (!is_loose)
-        continue;
+/**
+ * ostree_repo_prune_from_reachable:
+ * @self: Repo
+ * @options: Options controlling prune process
+ * @out_objects_total: (out): Number of objects found
+ * @out_objects_pruned: (out): Number of objects deleted
+ * @out_pruned_object_size_total: (out): Storage size in bytes of objects deleted
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Delete content from the repository.  This function is the "backend"
+ * half of the higher level ostree_repo_prune().  To use this function,
+ * you determine the root set yourself, and this function finds all other
+ * unreferenced objects and deletes them.
+ *
+ * Use this API when you want to perform more selective pruning - for example,
+ * retain all commits from a production branch, but just GC some history from
+ * your dev branch.
+ *
+ * The %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE flag may be specified to just determine
+ * statistics on objects that would be deleted, without actually deleting them.
+ */
+gboolean
+ostree_repo_prune_from_reachable (OstreeRepo        *self,
+                                  OstreeRepoPruneOptions *options,
+                                  gint              *out_objects_total,
+                                  gint              *out_objects_pruned,
+                                  guint64           *out_pruned_object_size_total,
+                                  GCancellable      *cancellable,
+                                  GError           **error)
+{
+  g_autoptr(GHashTable) objects = NULL;
 
-      if (!maybe_prune_loose_object (&data, flags, checksum, objtype,
-                                     cancellable, error))
-        goto out;
-    }
+  if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,
+                                 &objects, cancellable, error))
+    return FALSE;
 
-  if (!ostree_repo_prune_static_deltas (self, NULL, cancellable, error))
-    goto out;
-
-  if (!_ostree_repo_prune_tmp (self, cancellable, error))
-    goto out;
-
-  ret = TRUE;
-  *out_objects_total = (data.n_reachable_meta + data.n_unreachable_meta +
-                        data.n_reachable_content + data.n_unreachable_content);
-  *out_objects_pruned = (data.n_unreachable_meta + data.n_unreachable_content);
-  *out_pruned_object_size_total = data.freed_bytes;
- out:
-  if (data.reachable)
-    g_hash_table_unref (data.reachable);
-  return ret;
+  return repo_prune_internal (self, objects, options, out_objects_total,
+                              out_objects_pruned, out_pruned_object_size_total,
+                              cancellable, error);
 }

src/libostree/ostree-repo.h

@@ -970,6 +970,27 @@ gboolean ostree_repo_prune (OstreeRepo        *self,
                             GCancellable      *cancellable,
                             GError           **error);
 
+struct _OstreeRepoPruneOptions {
+  OstreeRepoPruneFlags flags;
+
+  GHashTable *reachable; /* Set<GVariant> (object names) */
+
+  gboolean unused_bools[6];
+  int unused_ints[6];
+  gpointer unused_ptrs[7];
+};
+
+typedef struct _OstreeRepoPruneOptions OstreeRepoPruneOptions;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_prune_from_reachable (OstreeRepo             *self,
+                                           OstreeRepoPruneOptions *options,
+                                           gint              *out_objects_total,
+                                           gint              *out_objects_pruned,
+                                           guint64           *out_pruned_object_size_total,
+                                           GCancellable           *cancellable,
+                                           GError              **error);
+
 /**
  * OstreeRepoPullFlags:
  * @OSTREE_REPO_PULL_FLAGS_NONE: No special options for pull

src/ostree/ot-builtin-prune.c

@@ -34,6 +34,7 @@ static gint opt_depth = -1;
 static gboolean opt_refs_only;
 static char *opt_delete_commit;
 static char *opt_keep_younger_than;
+static char **opt_retain_branch_depth;
 
 static GOptionEntry options[] = {
   { "no-prune", 0, 0, G_OPTION_ARG_NONE, &opt_no_prune, "Only display unreachable objects; don't delete", NULL },
@@ -42,6 +43,7 @@ static GOptionEntry options[] = {
   { "delete-commit", 0, 0, G_OPTION_ARG_STRING, &opt_delete_commit, "Specify a commit to delete", "COMMIT" },
   { "keep-younger-than", 0, 0, G_OPTION_ARG_STRING, &opt_keep_younger_than, "Prune all commits older than the specified date", "DATE" },
   { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" },
+  { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" },
   { NULL }
 };
 
@@ -82,87 +84,53 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can
 }
 
 static gboolean
-prune_commits_keep_younger_than_date (OstreeRepo *repo, const char *date, GCancellable *cancellable, GError **error)
+traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
+                            struct timespec *ts,
+                            GHashTable *reachable,
+                            GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GHashTable) refs = NULL;
-  g_autoptr(GHashTable) ref_heads = g_hash_table_new (g_str_hash, g_str_equal);
-  g_autoptr(GHashTable) objects = NULL;
-  GHashTableIter hash_iter;
-  gpointer key, value;
-  struct timespec ts;
-  gboolean ret = FALSE;
+  g_autofree char *next_checksum = g_strdup (checksum);
+  g_autoptr(GVariant) commit = NULL;
 
-  if (!parse_datetime (&ts, date, NULL))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Could not parse '%s'", date);
-      goto out;
-    }
-
-  if (!ot_enable_tombstone_commits (repo, error))
-    goto out;
-
-  if (!ostree_repo_list_refs (repo, NULL, &refs, cancellable, error))
-    goto out;
-
-  /* We used to prune the HEAD of a given ref by default, but that's
-   * broken for a few reasons.  One is that people may use branches as
-   * tags.  Second is that if we do it, we should be deleting the ref
-   * too, otherwise e.g. `summary -u` breaks trying to load it, etc.
+  /* This is the first commit in our loop, which has a ref pointing to it. We
+   * don't want to auto-prune it.
    */
-  g_hash_table_iter_init (&hash_iter, refs);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+  if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable,
+                                          cancellable, error))
+    return FALSE;
+
+  while (TRUE)
     {
-      /* Value is lifecycle bound to refs */
-      g_hash_table_add (ref_heads, (char*)value);
-    }
-
-  if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
-                                 cancellable, error))
-    goto out;
-
-  g_hash_table_iter_init (&hash_iter, objects);
-
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      const char *checksum;
-      OstreeObjectType objtype;
       guint64 commit_timestamp;
-      g_autoptr(GVariant) commit = NULL;
 
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+      if (!ostree_repo_load_variant_if_exists (repo, OSTREE_OBJECT_TYPE_COMMIT, next_checksum,
+                                               &commit, error))
+        return FALSE;
 
-      if (objtype != OSTREE_OBJECT_TYPE_COMMIT)
-        continue;
-
-      if (g_hash_table_contains (ref_heads, checksum))
-        continue;
-
-      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
-                                     &commit, error))
-        goto out;
+      if (!commit)
+        break; /* This commit was pruned, so we're done */
 
       commit_timestamp = ostree_commit_get_timestamp (commit);
-      if (commit_timestamp < ts.tv_sec)
+      /* Is this commit newer than our --keep-younger-than spec? */
+      if (commit_timestamp >= ts->tv_sec)
         {
-          if (opt_static_deltas_only)
-            {
-              if(!ostree_repo_prune_static_deltas (repo, checksum, cancellable, error))
-                goto out;
-            }
+          /* It's newer, traverse it */
+          if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable,
+                                                  cancellable, error))
+            return FALSE;
+
+          g_free (next_checksum);
+          next_checksum = ostree_commit_get_parent (commit);
+          if (next_checksum)
+            g_clear_pointer (&commit, (GDestroyNotify)g_variant_unref);
           else
-            {
-              if (!ostree_repo_delete_object (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum, cancellable, error))
-                goto out;
-            }
+            break; /* No parent, we're done */
         }
+      else
+        break; /* It's older than our spec, we're done */
     }
 
-  ret = TRUE;
-
- out:
-  return ret;
+  return TRUE;
 }
 
 gboolean
@@ -185,6 +153,9 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
   if (!opt_no_prune && !ostree_ensure_repo_writable (repo, error))
     goto out;
 
+  /* Special handling for explicit commit deletion here - we do this
+   * first.
+   */
   if (opt_delete_commit)
     {
       if (opt_no_prune)
@@ -200,26 +171,133 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
         else if (!delete_commit (repo, opt_delete_commit, cancellable, error))
           goto out;
     }
-  if (opt_keep_younger_than)
-    {
-      if (opt_no_prune)
-        {
-          ot_util_usage_error (context, "Cannot specify both --keep-younger-than and --no-prune", error);
-          goto out;
-        }
-      if (!prune_commits_keep_younger_than_date (repo, opt_keep_younger_than, cancellable, error))
-        goto out;
-    }
 
   if (opt_refs_only)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
   if (opt_no_prune)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE;
 
-  if (!ostree_repo_prune (repo, pruneflags, opt_depth,
-                          &n_objects_total, &n_objects_pruned, &objsize_total,
-                          cancellable, error))
-    goto out;
+  /* If no newer more complex options are specified, drop down to the original
+   * prune API - both to avoid code duplication, and to keep it run from the
+   * test suite.
+   */
+  if (!(opt_retain_branch_depth || opt_keep_younger_than))
+    {
+      if (!ostree_repo_prune (repo, pruneflags, opt_depth,
+                              &n_objects_total, &n_objects_pruned, &objsize_total,
+                              cancellable, error))
+        goto out;
+    }
+  else
+    {
+      g_autoptr(GHashTable) all_refs = NULL;
+      g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable ();
+      g_autoptr(GHashTable) retain_branch_depth = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+      struct timespec keep_younger_than_ts;
+      GHashTableIter hash_iter;
+      gpointer key, value;
+
+      /* Otherwise, the default is --refs-only; we set this just as a note */
+      opt_refs_only = TRUE;
+
+      if (opt_keep_younger_than)
+        {
+          if (!parse_datetime (&keep_younger_than_ts, opt_keep_younger_than, NULL))
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Could not parse '%s'", opt_keep_younger_than);
+              goto out;
+            }
+        }
+
+      for (char **iter = opt_retain_branch_depth; iter && *iter; iter++)
+        {
+          /* bd should look like BRANCH=DEPTH where DEPTH is an int */
+          const char *bd = *iter;
+          const char *eq = strchr (bd, '=');
+          const char *depthstr;
+          gint64 depth;
+          char *endptr;
+
+          if (!eq)
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid value %s, must specify BRANCH=DEPTH",
+                           bd);
+              goto out;
+            }
+          depthstr = eq + 1;
+          errno = EPERM;
+          depth = g_ascii_strtoll (depthstr, &endptr, 10);
+          if (depth == 0)
+            {
+              if (errno == EINVAL)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Out of range depth %s", depthstr);
+                  goto out;
+                }
+              else if (endptr == depthstr)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Invalid depth %s", depthstr);
+                  goto out;
+                }
+            }
+          g_hash_table_insert (retain_branch_depth, g_strndup (bd, eq - bd),
+                               GINT_TO_POINTER ((int)depth));
+        }
+
+      /* We start from the refs */
+      if (!ostree_repo_list_refs (repo, NULL, &all_refs,
+                                  cancellable, error))
+        return FALSE;
+
+      g_hash_table_iter_init (&hash_iter, all_refs);
+      while (g_hash_table_iter_next (&hash_iter, &key, &value))
+        {
+          const char *checksum = value;
+          gpointer depthp = g_hash_table_lookup (retain_branch_depth, key);
+          gint depth;
+
+          /* Here, we handle a spec like
+           * --retain-branch-depth=myos/x86_64/stable=-1
+           * --retain-branch-depth=myos/x86_64/dev=5
+           */
+          if (depthp)
+            depth = GPOINTER_TO_INT(depthp);
+          else if (opt_keep_younger_than)
+            {
+              if (!traverse_keep_younger_than (repo, checksum,
+                                               &keep_younger_than_ts,
+                                               reachable,
+                                               cancellable, error))
+                goto out;
+
+              /* Okay, we handled the younger-than case; the other
+               * two fall through to plain depth-based handling below.
+               */
+              continue;  /* Note again, we're skipping the below bit */
+            }
+          else
+            depth = opt_depth; /* No --retain-branch-depth for this branch, use
+                                  the global default */
+
+          g_debug ("Finding objects to keep for commit %s", checksum);
+          if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable,
+                                                  cancellable, error))
+            return FALSE;
+        }
+
+      { OstreeRepoPruneOptions opts = { pruneflags, reachable };
+        if (!ostree_repo_prune_from_reachable (repo, &opts,
+                                               &n_objects_total,
+                                               &n_objects_pruned,
+                                               &objsize_total,
+                                               cancellable, error))
+          goto out;
+      }
+    }
 
   formatted_freed_size = g_format_size_full (objsize_total, 0);