From 58b48424bcf854e6c4cbb4b529ae2266f17b5883 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 11 Mar 2016 12:39:32 +0100 Subject: [PATCH] pull: cache summary and summary.sig It allows an optimization to skip the download of the summary file if its .sig file is unchanged. Downloading the .sig file is much cheaper than downloading the summary file from repositories with many branches. https://bugzilla.gnome.org/show_bug.cgi?id=762973 Signed-off-by: Giuseppe Scrivano --- src/libostree/ostree-repo-private.h | 22 ++++- src/libostree/ostree-repo-pull.c | 145 +++++++++++++++++++++++++--- src/libostree/ostree-repo.c | 11 +++ 3 files changed, 165 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 463b3dd7..484a6ec8 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -33,6 +33,8 @@ G_BEGIN_DECLS #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" +#define _OSTREE_SUMMARY_CACHE_PATH "tmp/cache/summaries" + /** * OstreeRepo: * @@ -316,5 +318,23 @@ _ostree_repo_read_bare_fd (OstreeRepo *self, gboolean _ostree_repo_update_mtime (OstreeRepo *self, GError **error); - + +/* Load the summary from the cache if the provided .sig file is the same as the + cached version. */ +gboolean +_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, + const char *remote, + GBytes *summary_sig, + GBytes **summary, + GCancellable *cancellable, + GError **error); + +gboolean +_ostree_repo_cache_summary (OstreeRepo *self, + const char *remote, + GBytes *summary, + GBytes *summary_sig, + GCancellable *cancellable, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0be1b380..90878484 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -28,6 +28,7 @@ #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" #include "otutil.h" +#include "ot-fs-utils.h" #include @@ -1767,6 +1768,102 @@ ostree_repo_pull_one_dir (OstreeRepo *self, progress, cancellable, error); } +gboolean +_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, + const char *remote, + GBytes *summary_sig, + GBytes **summary, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig"); + + glnx_fd_close int prev_fd = -1; + g_autoptr(GBytes) old_sig_contents = NULL; + + if (!ot_openat_ignore_enoent (self->repo_dir_fd, summary_cache_sig_file, &prev_fd, error)) + goto out; + + if (prev_fd < 0) + { + ret = TRUE; + goto out; + } + + old_sig_contents = glnx_fd_readall_bytes (prev_fd, cancellable, error); + if (!old_sig_contents) + goto out; + + if (g_bytes_compare (old_sig_contents, summary_sig) == 0) + { + const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote); + glnx_fd_close int summary_fd = -1; + GBytes *summary_data; + + + summary_fd = openat (self->repo_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY); + if (summary_fd < 0) + { + if (errno == ENOENT) + { + (void) unlinkat (self->repo_dir_fd, summary_cache_sig_file, 0); + ret = TRUE; + goto out; + } + + glnx_set_error_from_errno (error); + goto out; + } + + summary_data = glnx_fd_readall_bytes (summary_fd, cancellable, error); + if (!summary_data) + goto out; + *summary = summary_data; + } + ret = TRUE; + + out: + return ret; +} + +gboolean +_ostree_repo_cache_summary (OstreeRepo *self, + const char *remote, + GBytes *summary, + GBytes *summary_sig, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote); + const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig"); + + if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, _OSTREE_SUMMARY_CACHE_PATH, 0775, cancellable, error)) + goto out; + + if (!glnx_file_replace_contents_at (self->repo_dir_fd, + summary_cache_file, + g_bytes_get_data (summary, NULL), + g_bytes_get_size (summary), + self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + goto out; + + if (!glnx_file_replace_contents_at (self->repo_dir_fd, + summary_cache_sig_file, + g_bytes_get_data (summary_sig, NULL), + g_bytes_get_size (summary_sig), + self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; + +} + /* Documented in ostree-repo.c */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -1995,15 +2092,36 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GVariant) refs = NULL; g_autoptr(GVariant) deltas = NULL; g_autoptr(GVariant) additional_metadata = NULL; - - if (!pull_data->summary) + gboolean summary_from_cache = FALSE; + + if (!pull_data->summary_data_sig) + { + uri = suburi_new (pull_data->base_uri, "summary.sig", NULL); + if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE, + &bytes_sig, cancellable, error)) + goto out; + soup_uri_free (uri); + } + + if (bytes_sig && !_ostree_repo_load_cache_summary_if_same_sig (self, + remote_name_or_baseurl, + bytes_sig, + &bytes_summary, + cancellable, + error)) + goto out; + + if (bytes_summary) + summary_from_cache = TRUE; + + if (!pull_data->summary && !bytes_summary) { uri = suburi_new (pull_data->base_uri, "summary", NULL); if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE, &bytes_summary, cancellable, error)) goto out; soup_uri_free (uri); - } + } if (!bytes_summary && pull_data->gpg_verify_summary) { @@ -2019,15 +2137,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } - if (bytes_summary) - { - uri = suburi_new (pull_data->base_uri, "summary.sig", NULL); - if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE, - &bytes_sig, cancellable, error)) - goto out; - soup_uri_free (uri); - } - if (!bytes_sig && pull_data->gpg_verify_summary) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -2044,6 +2153,18 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->summary_data_sig = g_bytes_ref (bytes_sig); } + + if (!summary_from_cache && bytes_summary && bytes_sig) + { + if (!_ostree_repo_cache_summary (self, + remote_name_or_baseurl, + bytes_summary, + bytes_sig, + cancellable, + error)) + goto out; + } + if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) { g_autoptr(GVariant) sig_variant = NULL; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 2398c46f..25cfcb80 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -1795,6 +1795,17 @@ repo_remote_fetch_summary (OstreeRepo *self, goto out; } + if (*out_summary && *out_signatures) + { + if (!_ostree_repo_cache_summary (self, + name, + *out_summary, + *out_signatures, + cancellable, + error)) + goto out; + } + ret = TRUE; out: