packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2205 from pwithnall/etags-and-last-modified

Browse Source 
Add support for ETag and Last-Modified headers for summary and summary.sig
This commit is contained in:
OpenShift Merge Robot authored and GitHub committed 2020-10-22 18:20:23 -04:00
2 parents 95a6d15145 0974a7faf1
commit f8f6464580
16 files changed
+1007 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/libostree/ostree-repo-pull.c
+264 -15
View File
@@ -46,6 +46,7 @@
#include <gio/gunixinputstream.h>
#include <sys/statvfs.h>
#include <sys/time.h>
#ifdef HAVE_LIBSYSTEMD
#include <systemd/sd-journal.h>
#endif
@@ -458,8 +459,9 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
g_autoptr(GBytes) bytes = NULL;
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
NULL, 0,
n_network_retries,
&bytes,
&bytes, NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
return FALSE;
@@ -965,7 +967,7 @@ content_fetch_on_complete (GObject *object,
OstreeObjectType objtype;
gboolean free_fetch_data = TRUE;
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
goto out;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -1105,7 +1107,7 @@ meta_fetch_on_complete (GObject *object,
g_debug ("fetch of %s%s complete", checksum_obj,
fetch_data->is_detached_meta ? " (detached)" : "");
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
@@ -1282,7 +1284,7 @@ static_deltapart_fetch_on_complete (GObject *object,
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
goto out;
/* Transfer ownership of the fd */
@@ -1994,7 +1996,7 @@ start_fetch (OtPullData *pull_data,
if (!is_meta && pull_data->trusted_http_direct)
flags |= OSTREE_FETCHER_REQUEST_LINKABLE;
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
obj_subpath, flags, expected_max_size,
obj_subpath, flags, NULL, 0, expected_max_size,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
@@ -2119,7 +2121,7 @@ start_fetch_deltapart (OtPullData *pull_data,
g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, 0, fetch->size,
deltapart_path, 0, NULL, 0, fetch->size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
@@ -2494,6 +2496,7 @@ on_superblock_fetched (GObject *src,
if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
res,
&delta_superblock_data,
NULL, NULL, NULL,
error))
{
if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@@ -2570,6 +2573,7 @@ start_fetch_delta_superblock (OtPullData *pull_data,
_ostree_fetcher_request_to_membuf (pull_data->fetcher,
pull_data->content_mirrorlist,
delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
OSTREE_MAX_METADATA_SIZE,
0, pull_data->cancellable,
on_superblock_fetched,
@@ -2699,6 +2703,64 @@ _ostree_repo_verify_summary (OstreeRepo *self,
return TRUE;
}
static void
_ostree_repo_load_cache_summary_properties (OstreeRepo *self,
const char *filename,
const char *extension,
char **out_etag,
guint64 *out_last_modified)
{
const char *file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", filename, extension);
glnx_autofd int fd = -1;
if (self->cache_dir_fd == -1)
return;
if (!glnx_openat_rdonly (self->cache_dir_fd, file, TRUE, &fd, NULL))
return;
if (out_etag != NULL)
{
g_autoptr(GBytes) etag_bytes = glnx_fgetxattr_bytes (fd, "user.etag", NULL);
if (etag_bytes != NULL)
{
const guint8 *buf;
gsize buf_len;
buf = g_bytes_get_data (etag_bytes, &buf_len);
/* Loosely validate against https://tools.ietf.org/html/rfc7232#section-2.3
* by checking there are no embedded nuls. */
for (gsize i = 0; i < buf_len; i++)
{
if (buf[i] == 0)
{
buf_len = 0;
break;
}
}
/* Nul-terminate and return */
if (buf_len > 0)
*out_etag = g_strndup ((const char *) buf, buf_len);
else
*out_etag = NULL;
}
else
*out_etag = NULL;
}
if (out_last_modified != NULL)
{
struct stat statbuf;
if (glnx_fstatat (fd, "", &statbuf, AT_EMPTY_PATH, NULL))
*out_last_modified = statbuf.st_mtim.tv_sec;
else
*out_last_modified = 0;
}
}
static gboolean
_ostree_repo_load_cache_summary_file (OstreeRepo *self,
const char *filename,
@@ -2774,11 +2836,38 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self,
return TRUE;
}
static void
store_file_cache_properties (int dir_fd,
const char *filename,
const char *etag,
guint64 last_modified)
{
glnx_autofd int fd = -1;
struct timespec time_vals[] =
{
{ .tv_sec = last_modified, .tv_nsec = UTIME_OMIT }, /* access, leave unchanged */
{ .tv_sec = last_modified, .tv_nsec = 0 }, /* modification */
};
if (!glnx_openat_rdonly (dir_fd, filename, TRUE, &fd, NULL))
return;
if (etag != NULL)
TEMP_FAILURE_RETRY (fsetxattr (fd, "user.etag", etag, strlen (etag), 0));
else
TEMP_FAILURE_RETRY (fremovexattr (fd, "user.etag"));
if (last_modified > 0)
TEMP_FAILURE_RETRY (futimens (fd, time_vals));
}
static gboolean
_ostree_repo_save_cache_summary_file (OstreeRepo *self,
const char *filename,
const char *extension,
GBytes *data,
const char *etag,
guint64 last_modified,
GCancellable *cancellable,
GError **error)
{
@@ -2799,6 +2888,9 @@ _ostree_repo_save_cache_summary_file (OstreeRepo *self,
cancellable, error))
return FALSE;
/* Store the caching properties. This is non-fatal on failure. */
store_file_cache_properties (self->cache_dir_fd, file, etag, last_modified);
return TRUE;
}
@@ -2807,16 +2899,24 @@ static gboolean
_ostree_repo_cache_summary (OstreeRepo *self,
const char *remote,
GBytes *summary,
const char *summary_etag,
guint64 summary_last_modified,
GBytes *summary_sig,
const char *summary_sig_etag,
guint64 summary_sig_last_modified,
GCancellable *cancellable,
GError **error)
{
if (!_ostree_repo_save_cache_summary_file (self, remote, NULL,
summary, cancellable, error))
summary,
summary_etag, summary_last_modified,
cancellable, error))
return FALSE;
if (!_ostree_repo_save_cache_summary_file (self, remote, ".sig",
summary_sig, cancellable, error))
summary_sig,
summary_sig_etag, summary_sig_last_modified,
cancellable, error))
return FALSE;
return TRUE;
@@ -2964,8 +3064,13 @@ _ostree_preload_metadata_file (OstreeRepo *self,
GPtrArray *mirrorlist,
const char *filename,
gboolean is_metalink,
const char *if_none_match,
guint64 if_modified_since,
guint n_network_retries,
GBytes **out_bytes,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GCancellable *cancellable,
GError **error)
{
@@ -3000,8 +3105,10 @@ _ostree_preload_metadata_file (OstreeRepo *self,
{
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
if_none_match, if_modified_since,
n_network_retries,
out_bytes, OSTREE_MAX_METADATA_SIZE,
out_bytes, out_not_modified, out_etag, out_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error);
}
}
@@ -3360,6 +3467,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
{
gboolean ret = FALSE;
g_autoptr(GBytes) bytes_summary = NULL;
gboolean summary_not_modified = FALSE;
g_autofree char *summary_etag = NULL;
guint64 summary_last_modified = 0;
g_autofree char *metalink_url_str = NULL;
g_autoptr(GHashTable) requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */
g_autoptr(GHashTable) commits_to_fetch = NULL;
@@ -3827,6 +3937,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
{
g_autoptr(GBytes) bytes_sig = NULL;
gboolean summary_sig_not_modified = FALSE;
g_autofree char *summary_sig_etag = NULL;
guint64 summary_sig_last_modified = 0;
gsize n;
g_autoptr(GVariant) refs = NULL;
g_autoptr(GVariant) deltas = NULL;
@@ -3855,14 +3968,47 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!bytes_sig)
{
g_autofree char *summary_sig_if_none_match = NULL;
guint64 summary_sig_if_modified_since = 0;
/* Load the summary.sig from the network, but send its ETag and
* Last-Modified from the on-disk cache (if it exists) to reduce the
* download size if nothing’s changed. */
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig",
&summary_sig_if_none_match, &summary_sig_if_modified_since);
g_clear_pointer (&summary_sig_etag, g_free);
summary_sig_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_sig_if_none_match, summary_sig_if_modified_since,
pull_data->n_network_retries,
&bytes_sig,
&summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary.sig from the cache. Also load summary, since
* `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
if (summary_sig_not_modified)
{
g_clear_pointer (&bytes_sig, g_bytes_unref);
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig",
&bytes_sig,
cancellable, error))
goto out;
if (!bytes_summary &&
!pull_data->remote_repo_local &&
!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
if (bytes_sig &&
@@ -3884,14 +4030,35 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!pull_data->summary && !bytes_summary)
{
g_autofree char *summary_if_none_match = NULL;
guint64 summary_if_modified_since = 0;
_ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL,
&summary_if_none_match, &summary_if_modified_since);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
summary_if_none_match, summary_if_modified_since,
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary from the cache. */
if (summary_not_modified)
{
g_clear_pointer (&bytes_summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL,
&bytes_summary,
cancellable, error))
goto out;
}
}
#ifndef OSTREE_DISABLE_GPGME
@@ -3930,7 +4097,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again */
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -3945,12 +4114,16 @@ ostree_repo_pull_with_options (OstreeRepo *self,
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@@ -3993,7 +4166,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
{
if (summary_from_cache)
{
/* The cached summary doesn't match, fetch a new one and verify again */
/* The cached summary doesn't match, fetch a new one and verify again.
* Don’t set the cache headers in the HTTP request, to force a
* full download. */
if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -4008,12 +4183,16 @@ ostree_repo_pull_with_options (OstreeRepo *self,
summary_from_cache = FALSE;
g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&summary_etag, g_free);
summary_last_modified = 0;
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0, /* no cache headers */
pull_data->n_network_retries,
&bytes_summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@@ -4033,6 +4212,8 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (bytes_summary)
{
pull_data->summary_data = g_bytes_ref (bytes_summary);
pull_data->summary_etag = g_strdup (summary_etag);
pull_data->summary_last_modified = summary_last_modified;
pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE);
if (!g_variant_is_normal_form (pull_data->summary))
@@ -4050,7 +4231,11 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
if (bytes_sig)
pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
{
pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
pull_data->summary_sig_etag = g_strdup (summary_sig_etag);
pull_data->summary_sig_last_modified = summary_sig_last_modified;
}
}
if (!summary_from_cache && bytes_summary && bytes_sig)
@@ -4059,7 +4244,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
!_ostree_repo_cache_summary (self,
remote_name_or_baseurl,
bytes_summary,
summary_etag, summary_last_modified,
bytes_sig,
summary_sig_etag, summary_sig_last_modified,
cancellable,
error))
goto out;
@@ -4494,6 +4681,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
cancellable, error))
goto out;
store_file_cache_properties (pull_data->repo->repo_dir_fd, "summary",
pull_data->summary_etag, pull_data->summary_last_modified);
if (pull_data->summary_data_sig)
{
buf = g_bytes_get_data (pull_data->summary_data_sig, &len);
@@ -4501,6 +4691,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
buf, len, replaceflag,
cancellable, error))
goto out;
store_file_cache_properties (pull_data->repo->repo_dir_fd, "summary.sig",
pull_data->summary_sig_etag, pull_data->summary_sig_last_modified);
}
}
@@ -4687,7 +4880,9 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary_etag, g_free);
g_clear_pointer (&pull_data->summary_data_sig, (GDestroyNotify) g_bytes_unref);
g_clear_pointer (&pull_data->summary_sig_etag, g_free);
g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
g_clear_pointer (&pull_data->static_delta_superblocks, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->commit_to_depth, (GDestroyNotify) g_hash_table_unref);
@@ -5499,8 +5694,10 @@ find_remotes_cb (GObject *obj,
mirrorlist,
commit_filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
data->n_network_retries,
&commit_bytes,
NULL, NULL, NULL,
0, /* no maximum size */
cancellable,
&error))
@@ -6114,6 +6311,16 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
g_autoptr(GPtrArray) mirrorlist = NULL;
const char *append_user_agent = NULL;
guint n_network_retries = DEFAULT_N_NETWORK_RETRIES;
gboolean summary_sig_not_modified = FALSE;
g_autofree char *summary_sig_if_none_match = NULL;
g_autofree char *summary_sig_etag = NULL;
gboolean summary_not_modified = FALSE;
g_autofree char *summary_if_none_match = NULL;
g_autofree char *summary_etag = NULL;
guint64 summary_sig_if_modified_since = 0;
guint64 summary_sig_last_modified = 0;
guint64 summary_if_modified_since = 0;
guint64 summary_last_modified = 0;
g_return_val_if_fail (OSTREE_REPO (self), FALSE);
g_return_val_if_fail (name != NULL, FALSE);
@@ -6159,22 +6366,48 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
&mirrorlist, cancellable, error))
return FALSE;
/* FIXME: Send the ETag from the cache with the request for summary.sig to
/* Send the ETag from the cache with the request for summary.sig to
* avoid downloading summary.sig unnecessarily. This won’t normally provide
* any benefits (but won’t do any harm) since summary.sig is typically 500B
* in size. But if a repository has multiple keys, the signature file will
* much benefit since summary.sig is typically 590B in size (vs a 0B HTTP 304
* response). But if a repository has multiple keys, the signature file will
* grow and this optimisation may be useful. */
_ostree_repo_load_cache_summary_properties (self, name, ".sig",
&summary_sig_if_none_match, &summary_sig_if_modified_since);
_ostree_repo_load_cache_summary_properties (self, name, NULL,
&summary_if_none_match, &summary_if_modified_since);
if (!_ostree_preload_metadata_file (self,
fetcher,
mirrorlist,
"summary.sig",
metalink_url_string ? TRUE : FALSE,
summary_sig_if_none_match, summary_sig_if_modified_since,
n_network_retries,
&signatures,
&summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified,
cancellable,
error))
return FALSE;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary.sig from the cache. Also load summary, since
* `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
if (summary_sig_not_modified)
{
g_clear_pointer (&signatures, g_bytes_unref);
g_clear_pointer (&summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, name, ".sig",
&signatures,
cancellable, error))
return FALSE;
if (!summary &&
!_ostree_repo_load_cache_summary_file (self, name, NULL,
&summary,
cancellable, error))
return FALSE;
}
if (signatures)
{
if (!_ostree_repo_load_cache_summary_if_same_sig (self,
@@ -6195,11 +6428,25 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
mirrorlist,
"summary",
metalink_url_string ? TRUE : FALSE,
summary_if_none_match, summary_if_modified_since,
n_network_retries,
&summary,
&summary_not_modified, &summary_etag, &summary_last_modified,
cancellable,
error))
return FALSE;
/* The server returned HTTP status 304 Not Modified, so we’re clear to
* load summary.sig from the cache. Also load summary, since
* `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */
if (summary_not_modified)
{
g_clear_pointer (&summary, g_bytes_unref);
if (!_ostree_repo_load_cache_summary_file (self, name, NULL,
&summary,
cancellable, error))
return FALSE;
}
}
if (!_ostree_repo_verify_summary (self, name,
@@ -6215,7 +6462,9 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
if (!_ostree_repo_cache_summary (self,
name,
summary,
summary_etag, summary_last_modified,
signatures,
summary_sig_etag, summary_sig_last_modified,
cancellable,
&temp_error))
{