From 4273e670eaec12b86a23608deeaf593799306b51 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 28 Jun 2017 22:19:15 -0400 Subject: [PATCH] Add "pull --localcache-repo" This is a lot like `git clone --reference`, but we chose "localcache" as the term "reference" is already used. The main use case I'm targeting this for is the Fedora Atomic Host installer case where we embed the repo content in the installer, but we may want to kickstart and download newer content. There, while we want to get a newer ref, we can still use the local repo as an object cache, since we have it sitting there in memory anyways. Another case is where one has a host ostree (say e.g. Fedora Atomic Workstation), and one wants to create a local archive mirror of FAH. Then one can use `pull --reference /ostree/repo` and pull the common objects (e.g. contents of `bash.rpm` etc.) Closes: https://github.com/ostreedev/ostree/issues/975 Closes: #982 Approved by: jlebon --- Makefile-tests.am | 1 + man/ostree-pull.xml | 10 +++ src/libostree/ostree-repo-pull.c | 102 ++++++++++++++++++++++++++++--- src/ostree/ot-builtin-pull.c | 5 ++ tests/test-pull-localcache.sh | 45 ++++++++++++++ 5 files changed, 153 insertions(+), 10 deletions(-) create mode 100755 tests/test-pull-localcache.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 348a8e4a..444b6636 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -78,6 +78,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-repeated.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ + tests/test-pull-localcache.sh \ tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index 24ab0b72..394a29c2 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -73,6 +73,16 @@ Boston, MA 02111-1307, USA. + + + + + Like git's clone --reference. Reuse the provided + OSTree repo as a local object cache of objects when doing HTTP fetches. + May be specified multiple times. + + + diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fb0c9e82..7346b299 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -64,6 +64,7 @@ typedef struct { GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ OstreeRepo *remote_repo_local; + GPtrArray *localcache_repos; /* Array */ GMainContext *main_context; GCancellable *cancellable; @@ -118,6 +119,9 @@ typedef struct { guint n_fetched_deltapart_fallbacks; guint n_fetched_metadata; guint n_fetched_content; + /* Objects from pull --localcache-repo */ + guint n_fetched_localcache_metadata; + guint n_fetched_localcache_content; int maxdepth; guint64 start_time; @@ -236,6 +240,8 @@ update_progress (gpointer user_data) "scanned-metadata", "u", n_scanned_metadata, "bytes-transferred", "t", bytes_transferred, "start-time", "t", start_time, + "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata, + "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content, /* Deltas */ "fetched-delta-parts", "u", pull_data->n_fetched_deltaparts, @@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data, */ static gboolean import_one_local_content_object (OtPullData *pull_data, + OstreeRepo *src_repo, const char *checksum, GCancellable *cancellable, GError **error) { - g_assert (pull_data->remote_repo_local); - const gboolean trusted = !pull_data->is_untrusted; if (trusted && !pull_data->is_bareuseronly_files) { - if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo, OSTREE_OBJECT_TYPE_FILE, checksum, trusted, cancellable, error)) @@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data, g_autoptr(GFileInfo) content_finfo = NULL; g_autoptr(GVariant) content_xattrs = NULL; - if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum, + if (!ostree_repo_load_file (src_repo, checksum, &content_input, &content_finfo, &content_xattrs, cancellable, error)) return FALSE; @@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData *pull_data, /* Is this a local repo? */ if (pull_data->remote_repo_local) { - if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error)) + if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local, + file_checksum, cancellable, error)) return FALSE; + + /* Note early loop continue */ + continue; } - else + + /* We're doing HTTP, but see if we have the object in a local cache first */ + gboolean did_import_from_cache_repo = FALSE; + if (pull_data->localcache_repos) { - /* In this case we're doing HTTP pulls */ - g_hash_table_add (pull_data->requested_content, file_checksum); - enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); - file_checksum = NULL; /* Transfer ownership */ + for (guint j = 0; j < pull_data->localcache_repos->len; j++) + { + OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum, + cancellable, error)) + return FALSE; + did_import_from_cache_repo = TRUE; + pull_data->n_fetched_localcache_content++; + break; + } } + if (did_import_from_cache_repo) + continue; /* Note early continue */ + + /* Not available locally, queue a HTTP request */ + g_hash_table_add (pull_data->requested_content, file_checksum); + enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); + file_checksum = NULL; /* Transfer ownership */ } g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1); @@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData *pull_data, is_stored = TRUE; is_requested = TRUE; } + /* Do we have any localcache repos? */ + else if (!is_stored && pull_data->localcache_repos) + { + for (guint i = 0; i < pull_data->localcache_repos->len; i++) + { + OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo, + objtype, tmp_checksum, + !pull_data->is_untrusted, + cancellable, error)) + return FALSE; + /* See comment above */ + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); + is_stored = TRUE; + is_requested = TRUE; + pull_data->n_fetched_localcache_metadata++; + break; + } + } if (!is_stored && !is_requested) { @@ -2865,6 +2924,7 @@ initiate_request (OtPullData *pull_data, * * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid + * * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean inherit_transaction = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ int i; + g_autofree char **opt_localcache_repos = NULL; if (options) { @@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers); (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); + (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)fetch_object_data_free); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); + if (opt_localcache_repos && *opt_localcache_repos) + { + pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref); + for (char **it = opt_localcache_repos; it && *it; it++) + { + const char *localcache_path = *it; + g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path); + g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file); + if (!ostree_repo_open (cacherepo, cancellable, error)) + goto out; + g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo)); + } + } + if (dir_to_pull != NULL || dirs_to_pull != NULL) { pull_data->dirs = g_ptr_array_new_with_free_func (g_free); @@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, else g_string_append_printf (buf, "%u metadata, %u content objects fetched", pull_data->n_fetched_metadata, pull_data->n_fetched_content); + if (pull_data->n_fetched_localcache_metadata || + pull_data->n_fetched_localcache_content) + g_string_append_printf (buf, " (%u meta, %u content local)", + pull_data->n_fetched_localcache_metadata, + pull_data->n_fetched_localcache_content); g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds", (guint64)(bytes_transferred / shift), @@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_object (&pull_data->fetcher); g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref); g_clear_object (&pull_data->cancellable); + g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&pull_data->remote_repo_local); g_free (pull_data->remote_name); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index edb09550..7898e107 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -41,6 +41,7 @@ static char* opt_cache_dir; static int opt_depth = 0; static int opt_frequency = 0; static char* opt_url; +static char** opt_localcache_repos; static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, @@ -57,6 +58,7 @@ static GOptionEntry options[] = { { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" }, { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, + { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" }, { NULL } }; @@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (opt_localcache_repos) + g_variant_builder_add (&builder, "{s@v}", "localcache-repos", + g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1))); if (opt_http_headers) { diff --git a/tests/test-pull-localcache.sh b/tests/test-pull-localcache.sh new file mode 100755 index 00000000..5f810acb --- /dev/null +++ b/tests/test-pull-localcache.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright (C) 2017 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" + +echo '1..1' + +cd ${test_tmpdir} +gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo" +ostree_repo_init repo --mode "archive" +ostree_repo_init repo-local --mode "archive" +for repo in repo{,-local}; do + ${CMD_PREFIX} ostree --repo=${repo} remote add --set=gpg-verify=false origin ${gnomerepo_url} +done + +# Pull the contents to our local cache +${CMD_PREFIX} ostree --repo=repo-local pull origin main +rm files -rf +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main files +echo anewfile > files/anewfile +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=dir=files + +${CMD_PREFIX} ostree --repo=repo pull -L repo-local origin main >out.txt +assert_file_has_content out.txt '3 metadata, 1 content objects fetched (4 meta, 5 content local)' +echo "ok pull --reference"