From 725d50a3b53de47deef53a7c9120d304cb9a625a Mon Sep 17 00:00:00 2001 From: Saqib Ali Date: Mon, 7 Feb 2022 10:53:08 -0500 Subject: [PATCH 1/4] src/ostree: Add --commit-only option to ostree prune Recently we have noticed exceedingly long execution times for multiple invocations of ostree prune. This is a result of calculating full reachability on each invocation. The --commit-only flag provides an alternative strategy. It will only traverse and delete commit objects to avoid the more expensive reachability calculations. This allows us to chain multiple --commit-only commands cheaply, and then follow with a more expensive ostree prune invocation at the end to clean up orphaned meta and content objects. --- Makefile-libostree.am | 6 +- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 5 ++ src/libostree/ostree-repo-prune.c | 109 +++++++++++++++++---------- src/libostree/ostree-repo-traverse.c | 74 +++++++++++++----- src/libostree/ostree-repo.h | 27 ++++++- src/ostree/ot-builtin-prune.c | 28 +++++-- 7 files changed, 179 insertions(+), 71 deletions(-) diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 02ae9c6a..9b48a308 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -171,9 +171,9 @@ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym # Uncomment this include when adding new development symbols. -# if BUILDOPT_IS_DEVEL_BUILD -# symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -# endif +if BUILDOPT_IS_DEVEL_BUILD +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index aa74c839..577ee808 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -451,6 +451,7 @@ ostree_repo_traverse_parents_get_commits ostree_repo_traverse_commit ostree_repo_traverse_commit_union ostree_repo_traverse_commit_union_with_parents +ostree_repo_traverse_commit_with_flags ostree_repo_commit_traverse_iter_cleanup ostree_repo_commit_traverse_iter_clear ostree_repo_commit_traverse_iter_get_dir diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 9168db73..74f9b9a9 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,11 @@ - uncomment the include in Makefile-libostree.am */ +LIBOSTREE_2022.2 { +global: + ostree_repo_traverse_commit_with_flags; +} LIBOSTREE_2021.5; + /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 175765fd..0c702dc9 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -48,6 +48,10 @@ maybe_prune_loose_object (OtPruneData *data, OstreeObjectType objtype; ostree_object_name_deserialize (key, &checksum, &objtype); + /* Return if we only want to delete commits and this object is not a commit object. */ + gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY; + if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT)) + goto exit; if (g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) reachable = TRUE; @@ -125,7 +129,11 @@ maybe_prune_loose_object (OtPruneData *data, else data->n_reachable_content++; } - + if (commit_only && (objtype != OSTREE_OBJECT_TYPE_COMMIT)) + { + g_debug ("Keeping object (not commit) %s.%s", checksum, + ostree_object_type_to_string (objtype)); + } return TRUE; } @@ -299,6 +307,52 @@ repo_prune_internal (OstreeRepo *self, return TRUE; } +static gboolean +traverse_reachable_internal (OstreeRepo *self, + OstreeRepoCommitTraverseFlags flags, + guint depth, + GHashTable *reachable, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(OstreeRepoAutoLock) lock = + ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error); + if (!lock) + return FALSE; + + /* Ignoring collections. */ + g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */ + + if (!ostree_repo_list_refs (self, NULL, &all_refs, + cancellable, error)) + return FALSE; + + GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum) + { + g_debug ("Finding objects to keep for commit %s", checksum); + if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable, + NULL, cancellable, error)) + return FALSE; + } + + /* Using collections. */ + g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */ + + if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs, + OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error)) + return FALSE; + + GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum) + { + g_debug ("Finding objects to keep for commit %s", checksum); + if (!ostree_repo_traverse_commit_with_flags (self, flags, checksum, depth, reachable, + NULL, cancellable, error)) + return FALSE; + } + + return TRUE; +} + /** * ostree_repo_traverse_reachable_refs: * @self: Repo @@ -319,42 +373,10 @@ ostree_repo_traverse_reachable_refs (OstreeRepo *self, GCancellable *cancellable, GError **error) { - g_autoptr(OstreeRepoAutoLock) lock = - ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_SHARED, cancellable, error); - if (!lock) - return FALSE; - - /* Ignoring collections. */ - g_autoptr(GHashTable) all_refs = NULL; /* (element-type utf8 utf8) */ - - if (!ostree_repo_list_refs (self, NULL, &all_refs, - cancellable, error)) - return FALSE; - - GLNX_HASH_TABLE_FOREACH_V (all_refs, const char*, checksum) - { - g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, - cancellable, error)) - return FALSE; - } - - /* Using collections. */ - g_autoptr(GHashTable) all_collection_refs = NULL; /* (element-type OstreeChecksumRef utf8) */ - - if (!ostree_repo_list_collection_refs (self, NULL, &all_collection_refs, - OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES, cancellable, error)) - return FALSE; - - GLNX_HASH_TABLE_FOREACH_V (all_collection_refs, const char*, checksum) - { - g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, - cancellable, error)) - return FALSE; - } - - return TRUE; + return traverse_reachable_internal (self, + OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, + depth, reachable, + cancellable, error); } /** @@ -401,6 +423,7 @@ ostree_repo_prune (OstreeRepo *self, g_autoptr(GHashTable) objects = NULL; gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY; + gboolean commit_only = flags & OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY; g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable (); @@ -409,9 +432,15 @@ ostree_repo_prune (OstreeRepo *self, * the deletion. */ + OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE; + if (commit_only) + traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY; + if (refs_only) { - if (!ostree_repo_traverse_reachable_refs (self, depth, reachable, cancellable, error)) + if (!traverse_reachable_internal (self, traverse_flags, + depth, reachable, + cancellable, error)) return FALSE; } @@ -432,8 +461,8 @@ ostree_repo_prune (OstreeRepo *self, continue; g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable, - cancellable, error)) + if (!ostree_repo_traverse_commit_with_flags (self, traverse_flags, checksum, depth, reachable, + NULL, cancellable, error)) return FALSE; } } diff --git a/src/libostree/ostree-repo-traverse.c b/src/libostree/ostree-repo-traverse.c index c5c204d7..5efed100 100644 --- a/src/libostree/ostree-repo-traverse.c +++ b/src/libostree/ostree-repo-traverse.c @@ -542,8 +542,9 @@ traverse_dirtree (OstreeRepo *repo, } /** - * ostree_repo_traverse_commit_union_with_parents: (skip) + * ostree_repo_traverse_commit_with_flags: (skip) * @repo: Repo + * @flags: change traversal behaviour according to these flags * @commit_checksum: ASCII SHA256 checksum * @maxdepth: Traverse this many parent commits, -1 for unlimited * @inout_reachable: Set of reachable objects @@ -561,15 +562,17 @@ traverse_dirtree (OstreeRepo *repo, * Since: 2018.5 */ gboolean -ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, - const char *commit_checksum, - int maxdepth, - GHashTable *inout_reachable, - GHashTable *inout_parents, - GCancellable *cancellable, - GError **error) +ostree_repo_traverse_commit_with_flags (OstreeRepo *repo, + OstreeRepoCommitTraverseFlags flags, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GHashTable *inout_parents, + GCancellable *cancellable, + GError **error) { g_autofree char *tmp_checksum = NULL; + gboolean commit_only = flags & OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY; while (TRUE) { @@ -603,16 +606,20 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, g_hash_table_add (inout_reachable, g_variant_ref (key)); - g_debug ("Traversing commit %s", commit_checksum); - ostree_cleanup_repo_commit_traverse_iter - OstreeRepoCommitTraverseIter iter = { 0, }; - if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit, - OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, - error)) - return FALSE; + /* Save time by skipping traversal of non-commit objects */ + if (!commit_only) + { + g_debug ("Traversing commit %s", commit_checksum); + ostree_cleanup_repo_commit_traverse_iter + OstreeRepoCommitTraverseIter iter = { 0, }; + if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit, + OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, + error)) + return FALSE; - if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error)) - return FALSE; + if (!traverse_iter (repo, &iter, key, inout_reachable, inout_parents, ignore_missing_dirs, cancellable, error)) + return FALSE; + } gboolean recurse = FALSE; if (maxdepth == -1 || maxdepth > 0) @@ -634,6 +641,39 @@ ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, return TRUE; } +/** + * ostree_repo_traverse_commit_union_with_parents: (skip) + * @repo: Repo + * @commit_checksum: ASCII SHA256 checksum + * @maxdepth: Traverse this many parent commits, -1 for unlimited + * @inout_reachable: Set of reachable objects + * @inout_parents: Map from object to parent object + * @cancellable: Cancellable + * @error: Error + * + * Update the set @inout_reachable containing all objects reachable + * from @commit_checksum, traversing @maxdepth parent commits. + * + * Additionally this constructs a mapping from each object to the parents + * of the object, which can be used to track which commits an object + * belongs to. + * + * Since: 2018.5 + */ +gboolean +ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GHashTable *inout_parents, + GCancellable *cancellable, + GError **error) +{ + return ostree_repo_traverse_commit_with_flags(repo, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, + commit_checksum, maxdepth, inout_reachable, inout_parents, + cancellable, error); +} + /** * ostree_repo_traverse_commit_union: (skip) * @repo: Repo diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 8a5c3b33..98571170 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1118,6 +1118,16 @@ typedef enum { OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0, } OstreeStaticDeltaIndexFlags; +/** + * OstreeRepoCommitTraverseFlags: + * @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE: No special options for traverse + * @OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY: Traverse and retrieve only commit objects. (Since: 2022.2) + */ +typedef enum { + OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0), + OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY = (1 << 1), +} OstreeRepoCommitTraverseFlags; + _OSTREE_PUBLIC gboolean ostree_repo_static_delta_reindex (OstreeRepo *repo, OstreeStaticDeltaIndexFlags flags, @@ -1171,6 +1181,7 @@ gboolean ostree_repo_traverse_commit_union (OstreeRepo *repo, GHashTable *inout_reachable, GCancellable *cancellable, GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo *repo, const char *commit_checksum, @@ -1180,6 +1191,16 @@ gboolean ostree_repo_traverse_commit_union_with_parents (OstreeRepo *rep GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_traverse_commit_with_flags (OstreeRepo *repo, + OstreeRepoCommitTraverseFlags flags, + const char *commit_checksum, + int maxdepth, + GHashTable *inout_reachable, + GHashTable *inout_parents, + GCancellable *cancellable, + GError **error); + struct _OstreeRepoCommitTraverseIter { gboolean initialized; /* 4 byte hole on 64 bit */ @@ -1189,10 +1210,6 @@ struct _OstreeRepoCommitTraverseIter { typedef struct _OstreeRepoCommitTraverseIter OstreeRepoCommitTraverseIter; -typedef enum { - OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE = (1 << 0) -} OstreeRepoCommitTraverseFlags; - _OSTREE_PUBLIC gboolean ostree_repo_commit_traverse_iter_init_commit (OstreeRepoCommitTraverseIter *iter, @@ -1245,11 +1262,13 @@ void ostree_repo_commit_traverse_iter_cleanup (void *p); * @OSTREE_REPO_PRUNE_FLAGS_NONE: No special options for pruning * @OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE: Don't actually delete objects * @OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY: Do not traverse individual commit objects, only follow refs + * @OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY: Only traverse commit objects. (Since 2022.2) */ typedef enum { OSTREE_REPO_PRUNE_FLAGS_NONE = 0, OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE = (1 << 0), OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY = (1 << 1), + OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY = (1 << 2), } OstreeRepoPruneFlags; _OSTREE_PUBLIC diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index d133bbd8..b2dd407a 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -35,6 +35,7 @@ static char *opt_delete_commit; static char *opt_keep_younger_than; static char **opt_retain_branch_depth; static char **opt_only_branches; +static gboolean opt_commit_only; /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -50,6 +51,7 @@ static GOptionEntry options[] = { { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" }, { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" }, { "only-branch", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_only_branches, "Only prune BRANCH (may be specified multiple times)", "BRANCH" }, + { "commit-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Only traverse and delete commit objects.", NULL }, { NULL } }; @@ -99,12 +101,15 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum, GCancellable *cancellable, GError **error) { g_autofree char *next_checksum = g_strdup (checksum); + OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE; + if (opt_commit_only) + traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY; /* This is the first commit in our loop, which has a ref pointing to it. We * don't want to auto-prune it. */ - if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable, - cancellable, error)) + if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, 0, reachable, + NULL, cancellable, error)) return FALSE; while (TRUE) @@ -121,8 +126,8 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum, if (commit_timestamp >= ts->tv_sec) { /* It's newer, traverse it */ - if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable, - cancellable, error)) + if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, next_checksum, 0, reachable, + NULL, cancellable, error)) return FALSE; g_free (next_checksum); @@ -183,6 +188,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY; if (opt_no_prune) pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE; + if (opt_commit_only) + pruneflags |= OSTREE_REPO_PRUNE_FLAGS_COMMIT_ONLY; /* If no newer more complex options are specified, drop down to the original * prune API - both to avoid code duplication, and to keep it run from the @@ -285,6 +292,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation /* Traverse each ref, and gather all objects pointed to by it up to a * specific depth (if configured). */ + OstreeRepoCommitTraverseFlags traverse_flags = OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE; + if (opt_commit_only) + /** We can avoid looking at all objects if --commit-only is specified **/ + traverse_flags |= OSTREE_REPO_COMMIT_TRAVERSE_FLAG_COMMIT_ONLY; g_hash_table_iter_init (&hash_iter, all_refs); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { @@ -316,8 +327,8 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation the global default */ g_debug ("Finding objects to keep for commit %s", checksum); - if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable, - cancellable, error)) + if (!ostree_repo_traverse_commit_with_flags (repo, traverse_flags, checksum, depth, reachable, + NULL, cancellable, error)) return FALSE; } @@ -333,7 +344,10 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation } g_autofree char *formatted_freed_size = g_format_size_full (objsize_total, 0); - g_print ("Total objects: %u\n", n_objects_total); + if (opt_commit_only) + g_print("Total (commit only) objects: %u\n", n_objects_total); + else + g_print ("Total objects: %u\n", n_objects_total); if (n_objects_pruned == 0) g_print ("No unreachable objects\n"); else if (pruneflags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE) From ce44b1907e6ef68dec4f80859e799891bfec2be4 Mon Sep 17 00:00:00 2001 From: Saqib Ali Date: Tue, 8 Feb 2022 09:08:44 -0500 Subject: [PATCH 2/4] man/prune, bash: Add --commit-only flag for ostree prune Update the man page and the auto-complete script to include the --commit-only flag --- bash/ostree | 1 + man/ostree-prune.xml | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/bash/ostree b/bash/ostree index c990462f..46363315 100644 --- a/bash/ostree +++ b/bash/ostree @@ -805,6 +805,7 @@ _ostree_prune() { --no-prune --refs-only --static-deltas-only + --commit-only " local options_with_args=" diff --git a/man/ostree-prune.xml b/man/ostree-prune.xml index e7e028ab..341ac7fb 100644 --- a/man/ostree-prune.xml +++ b/man/ostree-prune.xml @@ -120,6 +120,17 @@ License along with this library. If not, see . . + + + + + + Only traverse and delete commit objects. This leaves orphaned meta and + content objects, which can be cleaned up with another prune invocation. + One may want to use this option to cheaply delete multiple commits, + and then clean up with a more expensive prune at the end. + + From 18ab5361b99ccc88043d2f7d9ac8e99fcf19508c Mon Sep 17 00:00:00 2001 From: Saqib Ali Date: Tue, 22 Feb 2022 19:00:25 -0500 Subject: [PATCH 3/4] tests/test-prune.sh: expand testing for --commit-only Let's add additional tests to expand the test suite for the new --commit-only functionality. --- tests/test-prune.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/tests/test-prune.sh b/tests/test-prune.sh index e33ab1e7..b029b359 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -25,7 +25,7 @@ skip_without_user_xattrs setup_fake_remote_repo1 "archive" -echo '1..12' +echo '1..17' cd ${test_tmpdir} mkdir repo @@ -50,6 +50,12 @@ assert_repo_has_n_commits() { assert_streq "$(find ${repo}/objects -name '*.commit' | wc -l)" "${count}" } +assert_repo_has_n_non_commit_objects() { + repo=$1 + count=$2 + assert_streq "$(find ${repo}/objects -name '*.*' ! -name '*.commit' ! -name '*.tombstone-commit'| wc -l)" "${count}" +} + # Test --no-prune objectcount_orig=$(find repo/objects | wc -l) ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 --no-prune | tee noprune.txt @@ -294,3 +300,68 @@ if ${CMD_PREFIX} ostree --repo=repo prune --only-branch=BACON 2>err.txt; then fi assert_file_has_content err.txt "Refspec.*BACON.*not found" echo "ok --only-branch=BACON" + +# We will use the same principle as datesnap repo +# to create a snapshot to test --commit-only +rm -rf commit-only-test-repo +ostree_repo_init commit-only-test-repo --mode=archive +# Older commits w/ content objects +echo 'message' > tree/file.txt +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=stable -m test -s "new stable build 1" tree --timestamp="October 15 1985" +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=dev -m test -s "new dev build 1" tree --timestamp="October 15 1985" +# Commits without any content objects +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=stable -m test -s "new stable build 2" tree +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=dev -m test -s "new dev build 2" tree +# Commits with content objects +echo 'message2' > tree/file2.txt +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=stable -m test -s "new stable build 2" tree +${CMD_PREFIX} ostree --repo=commit-only-test-repo commit --branch=dev -m test -s "new dev build 2" tree +assert_repo_has_n_commits commit-only-test-repo 6 +reinitialize_commit_only_test_repo() { + rm repo -rf + ostree_repo_init repo --mode=archive + ${CMD_PREFIX} ostree --repo=repo pull-local --depth=-1 commit-only-test-repo +} +orig_obj_count=$(find commit-only-test-repo/objects -name '*.*' ! -name '*.commit' | wc -l) + +# --commit-only tests +# Test single branch +reinitialize_commit_only_test_repo +${CMD_PREFIX} ostree --repo=repo prune --commit-only --only-branch=dev --depth=0 +assert_repo_has_n_commits repo 4 +assert_repo_has_n_non_commit_objects repo ${orig_obj_count} +echo 'ok --commit-only and --only-branch' + +# Test multiple branches (and depth > 0) +reinitialize_commit_only_test_repo +${CMD_PREFIX} ostree --repo=repo prune --commit-only --refs-only --depth=1 +assert_repo_has_n_commits repo 4 +assert_repo_has_n_non_commit_objects repo ${orig_obj_count} +echo 'ok --commit-only and multiple branches (depth > 0)' + +# Test --delete-commit with --commit-only +reinitialize_commit_only_test_repo +# this commit does not have a parent commit +COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo log dev | grep ^commit | cut -f 2 -d' ' | tail -n 1) +${CMD_PREFIX} ostree --repo=repo prune --commit-only --delete-commit=$COMMIT_TO_DELETE +assert_repo_has_n_commits repo 5 +# We gain an extra +assert_repo_has_n_non_commit_objects repo ${orig_obj_count} +echo 'ok --commit-only and --delete-commit' + +# Test --delete-commit when it creates orphaned commits +reinitialize_commit_only_test_repo +# get the current HEAD's parent on dev branch +COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo log dev | grep ^commit | cut -f 2 -d' ' | head -n 2 | tail -n 1) +${CMD_PREFIX} ostree --repo=repo prune --commit-only --refs-only --delete-commit=$COMMIT_TO_DELETE +# we deleted a commit that orphaned another, so we lose two commits +assert_repo_has_n_commits repo 4 +assert_repo_has_n_non_commit_objects repo ${orig_obj_count} +echo 'ok --commit-only and --delete-commit with orphaned commits' + +# Test --keep-younger-than with --commit-only +reinitialize_commit_only_test_repo +${CMD_PREFIX} ostree --repo=repo prune --commit-only --keep-younger-than="1 week ago" +assert_repo_has_n_commits repo 4 +assert_repo_has_n_non_commit_objects repo ${orig_obj_count} +echo 'ok --commit-only and --keep-younger-than' From cf66eaccee710a53cb1e80a65b9b2f74e26e39c9 Mon Sep 17 00:00:00 2001 From: Saqib Ali Date: Thu, 24 Feb 2022 16:57:23 -0500 Subject: [PATCH 4/4] tests/test-prune.sh: Use TAP API Change tests to use the newer TAP API introduced in https://github.com/ostreedev/ostree/pull/2440 --- tests/test-prune.sh | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test-prune.sh b/tests/test-prune.sh index b029b359..20904f31 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -25,8 +25,6 @@ skip_without_user_xattrs setup_fake_remote_repo1 "archive" -echo '1..17' - cd ${test_tmpdir} mkdir repo ostree_repo_init repo @@ -150,7 +148,7 @@ if ${CMD_PREFIX} ostree --repo=repo prune --static-deltas-only --keep-younger-th fi assert_file_has_content_literal err.txt "--static-deltas-only requires --delete-commit" -echo "ok prune" +tap_ok prune rm repo -rf ostree_repo_init repo --mode=bare-user @@ -158,7 +156,7 @@ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 --commit-metadata-only origin test ${CMD_PREFIX} ostree --repo=repo prune -echo "ok prune with partial repo" +tap_ok prune with partial repo assert_has_n_objects() { find $1/objects -name '*.filez' | wc -l > object-count @@ -194,7 +192,7 @@ ${CMD_PREFIX} ostree --repo=repo refs --delete test ${CMD_PREFIX} ostree --repo=child-repo prune --refs-only --depth=0 assert_has_n_objects child-repo 3 -echo "ok prune with parent repo" +tap_ok prune with parent repo # Delete all the above since I can't be bothered to think about how new tests # would interact. We make a new repo test suite, then clone it @@ -239,7 +237,7 @@ $OSTREE fsck ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=5 assert_repo_has_n_commits repo 9 $OSTREE fsck -echo "ok retain branch depth and keep-younger-than" +tap_ok retain branch depth and keep-younger-than # Just stable branch ref, we should prune everything except the tip of dev, # so 8 stable + 1 dev = 9 @@ -248,7 +246,7 @@ ${CMD_PREFIX} ostree --repo=repo prune --depth=0 --retain-branch-depth=stable=-1 assert_repo_has_n_commits repo 9 $OSTREE fsck -echo "ok retain branch depth (alone)" +tap_ok retain branch depth [alone] # Test --only-branch with --depth=0; this should be exactly identical to the # above with a result of 9. @@ -256,13 +254,13 @@ reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=0 assert_repo_has_n_commits repo 9 $OSTREE fsck -echo "ok --only-branch --depth=0" +tap_ok --only-branch --depth=0 # Test --only-branch with --depth=1; should just add 1 to the above, for 10. reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=1 assert_repo_has_n_commits repo 10 -echo "ok --only-branch --depth=1" +tap_ok --only-branch --depth=1 # Test --only-branch with all branches reinitialize_datesnap_repo @@ -271,27 +269,27 @@ assert_repo_has_n_commits repo 2 reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable --depth=1 assert_repo_has_n_commits repo 4 -echo "ok --only-branch (all) --depth=1" +tap_ok --only-branch [all] --depth=1 # Test --only-branch and --retain-branch-depth overlap reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable --depth=0 \ --retain-branch-depth=stable=2 assert_repo_has_n_commits repo 4 -echo "ok --only-branch and --retain-branch-depth overlap" +tap_ok --only-branch and --retain-branch-depth overlap # Test --only-branch and --retain-branch-depth together reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=0 --retain-branch-depth=stable=2 assert_repo_has_n_commits repo 4 -echo "ok --only-branch and --retain-branch-depth together" +tap_ok --only-branch and --retain-branch-depth together # Test --only-branch with --keep-younger-than; this should be identical to the test # above for --retain-branch-depth=stable=-1 reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --only-branch=stable --keep-younger-than="1 week ago" assert_repo_has_n_commits repo 11 -echo "ok --only-branch --keep-younger-than" +tap_ok --only-branch --keep-younger-than # Test --only-branch with a nonexistent ref reinitialize_datesnap_repo @@ -299,7 +297,7 @@ if ${CMD_PREFIX} ostree --repo=repo prune --only-branch=BACON 2>err.txt; then fatal "we pruned BACON?" fi assert_file_has_content err.txt "Refspec.*BACON.*not found" -echo "ok --only-branch=BACON" +tap_ok --only-branch=BACON # We will use the same principle as datesnap repo # to create a snapshot to test --commit-only @@ -330,14 +328,14 @@ reinitialize_commit_only_test_repo ${CMD_PREFIX} ostree --repo=repo prune --commit-only --only-branch=dev --depth=0 assert_repo_has_n_commits repo 4 assert_repo_has_n_non_commit_objects repo ${orig_obj_count} -echo 'ok --commit-only and --only-branch' +tap_ok --commit-only and --only-branch # Test multiple branches (and depth > 0) reinitialize_commit_only_test_repo ${CMD_PREFIX} ostree --repo=repo prune --commit-only --refs-only --depth=1 assert_repo_has_n_commits repo 4 assert_repo_has_n_non_commit_objects repo ${orig_obj_count} -echo 'ok --commit-only and multiple branches (depth > 0)' +tap_ok --commit-only and multiple branches depth=1 # Test --delete-commit with --commit-only reinitialize_commit_only_test_repo @@ -347,7 +345,7 @@ ${CMD_PREFIX} ostree --repo=repo prune --commit-only --delete-commit=$COMMIT_TO_ assert_repo_has_n_commits repo 5 # We gain an extra assert_repo_has_n_non_commit_objects repo ${orig_obj_count} -echo 'ok --commit-only and --delete-commit' +tap_ok --commit-only and --delete-commit # Test --delete-commit when it creates orphaned commits reinitialize_commit_only_test_repo @@ -357,11 +355,12 @@ ${CMD_PREFIX} ostree --repo=repo prune --commit-only --refs-only --delete-commit # we deleted a commit that orphaned another, so we lose two commits assert_repo_has_n_commits repo 4 assert_repo_has_n_non_commit_objects repo ${orig_obj_count} -echo 'ok --commit-only and --delete-commit with orphaned commits' +tap_ok --commit-only and --delete-commit with orphaned commits # Test --keep-younger-than with --commit-only reinitialize_commit_only_test_repo ${CMD_PREFIX} ostree --repo=repo prune --commit-only --keep-younger-than="1 week ago" assert_repo_has_n_commits repo 4 assert_repo_has_n_non_commit_objects repo ${orig_obj_count} -echo 'ok --commit-only and --keep-younger-than' +tap_ok --commit-only and --keep-younger-than +tap_end