From 42c60effbe5037d674ce4b84979ccaf91edfad48 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 Feb 2016 13:28:03 -0500 Subject: [PATCH] pull: Support specifying exact commit to pull via branch@commit I don't know why we didn't do this a long time ago. This extends the pull API to allow grabbing a specific commit, and will set the branch to it. There's some support for this in the deploy engine, but there are a lot of reasons to support it for raw pulls (such as subset mirroring cases). In fact I'm thinking we should also have the override-version logic here too. NOTE: One thing I debated here is inventing a new syntax on the command line. Git doesn't seem to have this functionality (probably because it'd be rarely used). The '@' character at least doesn't conflict with anything. Anyways, I wanted this for some other test cases. Without this, writing tests that go between different commits is more awkward as one must generate the content in one repo, then pull downstream, then generate more content, then pull again. But now I can just keep track of commit IDs and do exactly what I want without synchronizing the tests. --- man/ostree-pull.xml | 27 +++++++++++++-- src/libostree/ostree-repo-pull.c | 57 +++++++++++++++++++++----------- src/libostree/ostree-repo.c | 1 + src/ostree/ot-builtin-pull.c | 38 +++++++++++++++++++-- tests/pull-test.sh | 15 +++++++++ 5 files changed, 114 insertions(+), 24 deletions(-) diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index d915bcd2..c419307e 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -111,14 +111,37 @@ Boston, MA 02111-1307, USA. Description - Downloads all content corresponding to the provided branch or commit from the given remote. + This command can retrieve just a specific commit, or go all + the way to performing a full mirror of the remote + repository. If no BRANCH is specified, + all branches are retrieved. + + + A special syntax in the @ character allows + specifying a specific commit to retrieve from a branch. This + + Example - $ ostree pull remote_name + $ ostree --repo=repo pull --depth=-1 --mirror remote_name + + Perform a complete mirror of the remote. (This is + likely most useful if your repository is also + archive-z2 mode) + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard + + Fetch the most recent commit to exampleos/x86_64/standard. + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard@98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4 + + Download the specific commit starting with + 98ea6e as if it was the latest commit for + exampleos/x86_64/standard. diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 750c68f5..88b468b0 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1762,6 +1762,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; char **refs_to_fetch = NULL; + char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; @@ -1776,9 +1777,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); + if (refs_to_fetch && override_commit_ids) + g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE); if (dir_to_pull) g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); @@ -2069,8 +2073,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else if (refs_to_fetch != NULL) { - char **strviter; - for (strviter = refs_to_fetch; *strviter; strviter++) + char **strviter = refs_to_fetch; + char **commitid_strviter = override_commit_ids ? override_commit_ids : NULL; + + while (*strviter) { const char *branch = *strviter; @@ -2081,8 +2087,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else { - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); + char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid); } + + strviter++; + if (commitid_strviter) + commitid_strviter++; } } else @@ -2109,28 +2120,36 @@ ostree_repo_pull_with_options (OstreeRepo *self, while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *branch = key; + const char *override_commitid = value; char *contents = NULL; - if (pull_data->summary) + /* Support specifying "" for an override commitid */ + if (override_commitid && *override_commitid) { - gsize commit_size = 0; - guint64 *malloced_size; - - if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) - goto out; - - malloced_size = g_new0 (guint64, 1); - *malloced_size = commit_size; - g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), g_strdup (override_commitid)); } - else + else { - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) - goto out; + if (pull_data->summary) + { + gsize commit_size = 0; + guint64 *malloced_size; + + if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) + goto out; + + malloced_size = g_new0 (guint64, 1); + *malloced_size = commit_size; + g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + } + else + { + if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + goto out; + } + /* Transfer ownership of contents */ + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } - - /* Transfer ownership of contents */ - g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } /* Create the state directory here - it's new with the commitpartial code, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 21362084..1e9673d1 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3784,6 +3784,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self, * * flags (i): An instance of #OstreeRepoPullFlags * * refs: (as): Array of string refs * * depth: (i): How far in the history to traverse; default is 0, -1 means infinite + * * override-commit-ids: (as): Array of specific commit IDs to fetch for refs */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 6aae8205..63dbc768 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -34,7 +34,7 @@ static gboolean opt_disable_static_deltas; static char* opt_subpath; static int opt_depth = 0; - static GOptionEntry options[] = { +static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, @@ -70,6 +70,7 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** OstreeRepoPullFlags pullflags = 0; GSConsole *console = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; + g_autoptr(GPtrArray) override_commit_ids = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; gulong signal_handler_id = 0; @@ -102,9 +103,36 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (argc > 2) { int i; - refs_to_fetch = g_ptr_array_new (); + refs_to_fetch = g_ptr_array_new_with_free_func (g_free); + for (i = 2; i < argc; i++) - g_ptr_array_add (refs_to_fetch, argv[i]); + { + const char *at = strrchr (argv[i], '@'); + + if (at) + { + guint j; + const char *override_commit_id = at + 1; + + if (!ostree_validate_checksum_string (override_commit_id, error)) + goto out; + + if (!override_commit_ids) + override_commit_ids = g_ptr_array_new_with_free_func (g_free); + + /* Backfill */ + for (j = 2; j < i; i++) + g_ptr_array_add (override_commit_ids, g_strdup ("")); + + g_ptr_array_add (override_commit_ids, g_strdup (override_commit_id)); + g_ptr_array_add (refs_to_fetch, g_strndup (argv[i], at - argv[i])); + } + else + { + g_ptr_array_add (refs_to_fetch, g_strdup (argv[i])); + } + } + g_ptr_array_add (refs_to_fetch, NULL); } } @@ -147,6 +175,10 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "disable-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_disable_static_deltas))); + if (override_commit_ids) + g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", + g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (!ostree_repo_pull_with_options (repo, remote, g_variant_builder_end (&builder), progress, cancellable, error)) goto out; diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 6f0b651a..2233b905 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -72,6 +72,21 @@ $OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta assert_file_has_content main-meta "HANCOCK" echo "ok pull detached metadata" +cd ${test_tmpdir} +mkdir parentpullrepo +${CMD_PREFIX} ostree --repo=parentpullrepo init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=parentpullrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo +parent_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main@${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo fsck +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${rev} +echo "ok pull specific commit" + cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main