diff --git a/Makefile-tests.am b/Makefile-tests.am index 348a8e4a..444b6636 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -78,6 +78,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-repeated.sh \ tests/test-pull-untrusted.sh \ tests/test-pull-override-url.sh \ + tests/test-pull-localcache.sh \ tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index 24ab0b72..394a29c2 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -73,6 +73,16 @@ Boston, MA 02111-1307, USA. + + + + + Like git's clone --reference. Reuse the provided + OSTree repo as a local object cache of objects when doing HTTP fetches. + May be specified multiple times. + + + diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fb0c9e82..7346b299 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -64,6 +64,7 @@ typedef struct { GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ OstreeRepo *remote_repo_local; + GPtrArray *localcache_repos; /* Array */ GMainContext *main_context; GCancellable *cancellable; @@ -118,6 +119,9 @@ typedef struct { guint n_fetched_deltapart_fallbacks; guint n_fetched_metadata; guint n_fetched_content; + /* Objects from pull --localcache-repo */ + guint n_fetched_localcache_metadata; + guint n_fetched_localcache_content; int maxdepth; guint64 start_time; @@ -236,6 +240,8 @@ update_progress (gpointer user_data) "scanned-metadata", "u", n_scanned_metadata, "bytes-transferred", "t", bytes_transferred, "start-time", "t", start_time, + "metadata-fetched-localcache", "u", pull_data->n_fetched_localcache_metadata, + "content-fetched-localcache", "u", pull_data->n_fetched_localcache_content, /* Deltas */ "fetched-delta-parts", "u", pull_data->n_fetched_deltaparts, @@ -606,16 +612,15 @@ validate_bareuseronly_mode (OtPullData *pull_data, */ static gboolean import_one_local_content_object (OtPullData *pull_data, + OstreeRepo *src_repo, const char *checksum, GCancellable *cancellable, GError **error) { - g_assert (pull_data->remote_repo_local); - const gboolean trusted = !pull_data->is_untrusted; if (trusted && !pull_data->is_bareuseronly_files) { - if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, src_repo, OSTREE_OBJECT_TYPE_FILE, checksum, trusted, cancellable, error)) @@ -630,7 +635,7 @@ import_one_local_content_object (OtPullData *pull_data, g_autoptr(GFileInfo) content_finfo = NULL; g_autoptr(GVariant) content_xattrs = NULL; - if (!ostree_repo_load_file (pull_data->remote_repo_local, checksum, + if (!ostree_repo_load_file (src_repo, checksum, &content_input, &content_finfo, &content_xattrs, cancellable, error)) return FALSE; @@ -709,16 +714,43 @@ scan_dirtree_object (OtPullData *pull_data, /* Is this a local repo? */ if (pull_data->remote_repo_local) { - if (!import_one_local_content_object (pull_data, file_checksum, cancellable, error)) + if (!import_one_local_content_object (pull_data, pull_data->remote_repo_local, + file_checksum, cancellable, error)) return FALSE; + + /* Note early loop continue */ + continue; } - else + + /* We're doing HTTP, but see if we have the object in a local cache first */ + gboolean did_import_from_cache_repo = FALSE; + if (pull_data->localcache_repos) { - /* In this case we're doing HTTP pulls */ - g_hash_table_add (pull_data->requested_content, file_checksum); - enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); - file_checksum = NULL; /* Transfer ownership */ + for (guint j = 0; j < pull_data->localcache_repos->len; j++) + { + OstreeRepo *localcache_repo = pull_data->localcache_repos->pdata[j]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (localcache_repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!import_one_local_content_object (pull_data, localcache_repo, file_checksum, + cancellable, error)) + return FALSE; + did_import_from_cache_repo = TRUE; + pull_data->n_fetched_localcache_content++; + break; + } } + if (did_import_from_cache_repo) + continue; /* Note early continue */ + + /* Not available locally, queue a HTTP request */ + g_hash_table_add (pull_data->requested_content, file_checksum); + enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, path, FALSE, FALSE); + file_checksum = NULL; /* Transfer ownership */ } g_autoptr(GVariant) dirs_variant = g_variant_get_child_value (tree, 1); @@ -1527,6 +1559,33 @@ scan_one_metadata_object_c (OtPullData *pull_data, is_stored = TRUE; is_requested = TRUE; } + /* Do we have any localcache repos? */ + else if (!is_stored && pull_data->localcache_repos) + { + for (guint i = 0; i < pull_data->localcache_repos->len; i++) + { + OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i]; + gboolean localcache_repo_has_obj; + + if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum, + &localcache_repo_has_obj, cancellable, error)) + return FALSE; + if (!localcache_repo_has_obj) + continue; + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, refd_repo, + objtype, tmp_checksum, + !pull_data->is_untrusted, + cancellable, error)) + return FALSE; + /* See comment above */ + if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum)); + is_stored = TRUE; + is_requested = TRUE; + pull_data->n_fetched_localcache_metadata++; + break; + } + } if (!is_stored && !is_requested) { @@ -2865,6 +2924,7 @@ initiate_request (OtPullData *pull_data, * * inherit-transaction (b): Don't initiate, finish or abort a transaction, useful to do multiple pulls in one transaction. * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid + * * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -2903,6 +2963,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean inherit_transaction = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ int i; + g_autofree char **opt_localcache_repos = NULL; if (options) { @@ -2929,6 +2990,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers); (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); + (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); @@ -2993,6 +3055,20 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)fetch_object_data_free); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); + if (opt_localcache_repos && *opt_localcache_repos) + { + pull_data->localcache_repos = g_ptr_array_new_with_free_func (g_object_unref); + for (char **it = opt_localcache_repos; it && *it; it++) + { + const char *localcache_path = *it; + g_autoptr(GFile) localcache_file = g_file_new_for_path (localcache_path); + g_autoptr(OstreeRepo) cacherepo = ostree_repo_new (localcache_file); + if (!ostree_repo_open (cacherepo, cancellable, error)) + goto out; + g_ptr_array_add (pull_data->localcache_repos, g_steal_pointer (&cacherepo)); + } + } + if (dir_to_pull != NULL || dirs_to_pull != NULL) { pull_data->dirs = g_ptr_array_new_with_free_func (g_free); @@ -3722,6 +3798,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, else g_string_append_printf (buf, "%u metadata, %u content objects fetched", pull_data->n_fetched_metadata, pull_data->n_fetched_content); + if (pull_data->n_fetched_localcache_metadata || + pull_data->n_fetched_localcache_content) + g_string_append_printf (buf, " (%u meta, %u content local)", + pull_data->n_fetched_localcache_metadata, + pull_data->n_fetched_localcache_content); g_string_append_printf (buf, "; %" G_GUINT64_FORMAT " %s transferred in %u seconds", (guint64)(bytes_transferred / shift), @@ -3769,6 +3850,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_object (&pull_data->fetcher); g_clear_pointer (&pull_data->extra_headers, (GDestroyNotify)g_variant_unref); g_clear_object (&pull_data->cancellable); + g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&pull_data->remote_repo_local); g_free (pull_data->remote_name); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index edb09550..7898e107 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -41,6 +41,7 @@ static char* opt_cache_dir; static int opt_depth = 0; static int opt_frequency = 0; static char* opt_url; +static char** opt_localcache_repos; static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, @@ -57,6 +58,7 @@ static GOptionEntry options[] = { { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" }, { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, + { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" }, { NULL } }; @@ -281,6 +283,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (opt_localcache_repos) + g_variant_builder_add (&builder, "{s@v}", "localcache-repos", + g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1))); if (opt_http_headers) { diff --git a/tests/test-pull-localcache.sh b/tests/test-pull-localcache.sh new file mode 100755 index 00000000..5f810acb --- /dev/null +++ b/tests/test-pull-localcache.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright (C) 2017 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive" + +echo '1..1' + +cd ${test_tmpdir} +gnomerepo_url="$(cat httpd-address)/ostree/gnomerepo" +ostree_repo_init repo --mode "archive" +ostree_repo_init repo-local --mode "archive" +for repo in repo{,-local}; do + ${CMD_PREFIX} ostree --repo=${repo} remote add --set=gpg-verify=false origin ${gnomerepo_url} +done + +# Pull the contents to our local cache +${CMD_PREFIX} ostree --repo=repo-local pull origin main +rm files -rf +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main files +echo anewfile > files/anewfile +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=dir=files + +${CMD_PREFIX} ostree --repo=repo pull -L repo-local origin main >out.txt +assert_file_has_content out.txt '3 metadata, 1 content objects fetched (4 meta, 5 content local)' +echo "ok pull --reference"