From 407c683524bb362cbc39b8445015e4b06e5296a5 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Thu, 24 Sep 2020 17:49:48 +0100 Subject: [PATCH] libostree: Add support for ETag and Last-Modified headers Add support in the soup and curl fetchers to send the `If-None-Match` and `If-Modified-Since` request headers, and pass on the `ETag` and `Last-Modified` response headers. This currently introduces no functional changes, but once call sites provide the appropriate integration, this will allow HTTP caching to happen with requests (typically with metadata requests, where the data is not immutable due to being content-addressed). That should reduce bandwidth requirements. Signed-off-by: Philip Withnall --- src/libostree/ostree-fetcher-curl.c | 251 +++++++++++++++++++++++++++- src/libostree/ostree-fetcher-soup.c | 81 ++++++++- src/libostree/ostree-fetcher-util.c | 43 ++++- src/libostree/ostree-fetcher-util.h | 10 ++ src/libostree/ostree-fetcher.h | 10 ++ src/libostree/ostree-metalink.c | 5 +- src/libostree/ostree-repo-pull.c | 29 +++- 7 files changed, 410 insertions(+), 19 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 0ce3ff00..975508eb 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -99,10 +99,16 @@ struct FetcherRequest { guint64 current_size; guint64 max_size; OstreeFetcherRequestFlags flags; + struct curl_slist *req_headers; + char *if_none_match; /* request ETag */ + guint64 if_modified_since; /* seconds since the epoch */ gboolean is_membuf; GError *caught_write_error; GLnxTmpfile tmpf; GString *output_buf; + gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ + char *out_etag; /* response ETag */ + guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */ CURL *easy; char error[CURL_ERROR_SIZE]; @@ -335,7 +341,17 @@ check_multi_info (OstreeFetcher *fetcher) else { curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response); - if (!is_file && !(response >= 200 && response < 300)) + + if (!is_file && response == 304 && + (req->if_none_match != NULL || req->if_modified_since > 0)) + { + /* Version on the server is unchanged from the version we have + * cached locally; report this as an out-argument, a zero-length + * response buffer, and no error. */ + req->out_not_modified = TRUE; + } + + if (!is_file && !(response >= 200 && response < 300) && response != 304) { GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response); @@ -575,6 +591,175 @@ write_cb (void *ptr, size_t size, size_t nmemb, void *data) return realsize; } +/* @buf must already be known to be long enough */ +static gboolean +parse_uint (const char *buf, + guint n_digits, + guint min, + guint max, + guint *out) +{ + guint64 number; + const char *end_ptr = NULL; + gint saved_errno = 0; + + g_return_val_if_fail (n_digits == 2 || n_digits == 4, FALSE); + g_return_val_if_fail (out != NULL, FALSE); + + errno = 0; + number = g_ascii_strtoull (buf, (gchar **)&end_ptr, 10); + saved_errno = errno; + + if (!g_ascii_isdigit (buf[0]) || + saved_errno != 0 || + end_ptr == NULL || + end_ptr != buf + n_digits || + number < min || + number > max) + return FALSE; + + *out = number; + return TRUE; +} + +/* Locale-independent parsing for RFC 2616 date/times. + * + * Reference: https://tools.ietf.org/html/rfc2616#section-3.3.1 + * + * Syntax: + * , :: GMT + * + * Note that this only accepts the full-year and GMT formats specified by + * RFC 1123. It doesn’t accept RFC 850 or asctime formats. + * + * Example: + * Wed, 21 Oct 2015 07:28:00 GMT + */ +static GDateTime * +parse_rfc2616_date_time (const char *buf, + size_t len) +{ + guint day_int, year_int, hour_int, minute_int, second_int; + const char *day_names[] = + { + "Mon", + "Tue", + "Wed", + "Thu", + "Fri", + "Sat", + "Sun", + }; + size_t day_name_index; + const char *month_names[] = + { + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", + }; + size_t month_name_index; + + if (len != 29) + return NULL; + + const char *day_name = buf; + const char *day = buf + 5; + const char *month_name = day + 3; + const char *year = month_name + 4; + const char *hour = year + 5; + const char *minute = hour + 3; + const char *second = minute + 3; + const char *tz = second + 3; + + for (day_name_index = 0; day_name_index < G_N_ELEMENTS (day_names); day_name_index++) + { + if (strncmp (day_names[day_name_index], day_name, 3) == 0) + break; + } + if (day_name_index >= G_N_ELEMENTS (day_names)) + return NULL; + /* don’t validate whether the day_name matches the rest of the date */ + if (*(day_name + 3) != ',' || *(day_name + 4) != ' ') + return NULL; + if (!parse_uint (day, 2, 1, 31, &day_int)) + return NULL; + if (*(day + 2) != ' ') + return NULL; + for (month_name_index = 0; month_name_index < G_N_ELEMENTS (month_names); month_name_index++) + { + if (strncmp (month_names[month_name_index], month_name, 3) == 0) + break; + } + if (month_name_index >= G_N_ELEMENTS (month_names)) + return NULL; + if (*(month_name + 3) != ' ') + return NULL; + if (!parse_uint (year, 4, 0, 9999, &year_int)) + return NULL; + if (*(year + 4) != ' ') + return NULL; + if (!parse_uint (hour, 2, 0, 23, &hour_int)) + return NULL; + if (*(hour + 2) != ':') + return NULL; + if (!parse_uint (minute, 2, 0, 59, &minute_int)) + return NULL; + if (*(minute + 2) != ':') + return NULL; + if (!parse_uint (second, 2, 0, 60, &second_int)) /* allow leap seconds */ + return NULL; + if (*(second + 2) != ' ') + return NULL; + if (strncmp (tz, "GMT", 3) != 0) + return NULL; + + return g_date_time_new_utc (year_int, month_name_index + 1, day_int, + hour_int, minute_int, second_int); +} + +/* CURLOPT_HEADERFUNCTION */ +static size_t +response_header_cb (const char *buffer, size_t size, size_t n_items, void *user_data) +{ + const size_t real_size = size * n_items; + GTask *task = G_TASK (user_data); + FetcherRequest *req; + + /* libcurl says that @size is always 1, but let’s check + * See https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION.html */ + g_assert (size == 1); + + req = g_task_get_task_data (task); + + const char *etag_header = "ETag: "; + const char *last_modified_header = "Last-Modified: "; + + if (real_size > strlen (etag_header) && + strncasecmp (buffer, etag_header, strlen (etag_header)) == 0) + { + g_clear_pointer (&req->out_etag, g_free); + req->out_etag = g_strstrip (g_strdup (buffer + strlen (etag_header))); + } + else if (real_size > strlen (last_modified_header) && + strncasecmp (buffer, last_modified_header, strlen (last_modified_header)) == 0) + { + g_autofree char *lm_buf = g_strstrip (g_strdup (buffer + strlen (last_modified_header))); + g_autoptr(GDateTime) dt = parse_rfc2616_date_time (lm_buf, strlen (lm_buf)); + req->out_last_modified = (dt != NULL) ? g_date_time_to_unix (dt) : 0; + } + + return real_size; +} + /* CURLOPT_PROGRESSFUNCTION */ static int prog_cb (void *p, double dltotal, double dlnow, double ult, double uln) @@ -600,6 +785,9 @@ request_unref (FetcherRequest *req) glnx_tmpfile_clear (&req->tmpf); if (req->output_buf) g_string_free (req->output_buf, TRUE); + g_free (req->if_none_match); + g_free (req->out_etag); + g_clear_pointer (&req->req_headers, (GDestroyNotify)curl_slist_free_all); curl_easy_cleanup (req->easy); g_free (req); @@ -721,8 +909,29 @@ initiate_next_curl_request (FetcherRequest *req, curl_easy_setopt (req->easy, CURLOPT_USERAGENT, self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING); - if (self->extra_headers) - curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers); + + /* Set caching request headers */ + if (req->if_none_match != NULL) + { + g_autofree char *if_none_match = g_strconcat ("If-None-Match: ", req->if_none_match, NULL); + req->req_headers = curl_slist_append (req->req_headers, if_none_match); + } + + if (req->if_modified_since > 0) + { + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (req->if_modified_since); + g_autofree char *mod_date = g_date_time_format (date_time, "If-Modified-Since: %a, %d %b %Y %H:%M:%S %Z"); + + req->req_headers = curl_slist_append (req->req_headers, mod_date); + } + + /* Append a copy of @extra_headers to @req_headers, as the former could change + * between requests or while a request is in flight */ + for (const struct curl_slist *l = self->extra_headers; l != NULL; l = l->next) + req->req_headers = curl_slist_append (req->req_headers, l->data); + + if (req->req_headers != NULL) + curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, req->req_headers); if (self->cookie_jar_path) { @@ -796,6 +1005,7 @@ initiate_next_curl_request (FetcherRequest *req, } curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb); + curl_easy_setopt (req->easy, CURLOPT_HEADERFUNCTION, response_header_cb); if (g_getenv ("OSTREE_DEBUG_HTTP")) curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L); curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error); @@ -815,6 +1025,7 @@ initiate_next_curl_request (FetcherRequest *req, /* closure bindings -> task */ curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task); curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task); + curl_easy_setopt (req->easy, CURLOPT_HEADERDATA, task); curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task); CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy); @@ -826,6 +1037,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, gboolean is_membuf, guint64 max_size, int priority, @@ -859,6 +1072,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self, req->filename = g_strdup (filename); req->max_size = max_size; req->flags = flags; + req->if_none_match = g_strdup (if_none_match); + req->if_modified_since = if_modified_since; req->is_membuf = is_membuf; /* We'll allocate the tmpfile on demand, so we handle * file I/O errors just in the write func. @@ -882,13 +1097,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, FALSE, max_size, priority, cancellable, callback, user_data); } @@ -897,6 +1115,9 @@ gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, GAsyncResult *result, GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error) { g_return_val_if_fail (g_task_is_valid (result, self), FALSE); @@ -912,6 +1133,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, *out_tmpf = req->tmpf; req->tmpf.initialized = FALSE; /* Transfer ownership */ + if (out_not_modified != NULL) + *out_not_modified = req->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&req->out_etag); + if (out_last_modified != NULL) + *out_last_modified = req->out_last_modified; + return TRUE; } @@ -920,13 +1148,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, TRUE, max_size, priority, cancellable, callback, user_data); } @@ -935,6 +1166,9 @@ gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, GAsyncResult *result, GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error) { GTask *task; @@ -955,6 +1189,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, g_assert (out_buf); *out_buf = ret; + if (out_not_modified != NULL) + *out_not_modified = req->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&req->out_etag); + if (out_last_modified != NULL) + *out_last_modified = req->out_last_modified; + return TRUE; } diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index 970ac7a4..52fe5fc3 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -90,9 +90,14 @@ typedef struct { gboolean is_membuf; OstreeFetcherRequestFlags flags; + char *if_none_match; /* request ETag */ + guint64 if_modified_since; /* seconds since the epoch */ GInputStream *request_body; GLnxTmpfile tmpf; GOutputStream *out_stream; + gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */ + char *out_etag; /* response ETag */ + guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */ guint64 max_size; guint64 current_size; @@ -196,8 +201,10 @@ pending_uri_unref (OstreeFetcherPendingURI *pending) g_free (pending->filename); g_clear_object (&pending->request); g_clear_object (&pending->request_body); + g_free (pending->if_none_match); glnx_tmpfile_clear (&pending->tmpf); g_clear_object (&pending->out_stream); + g_free (pending->out_etag); g_free (pending); } @@ -431,6 +438,22 @@ create_pending_soup_request (OstreeFetcherPendingURI *pending, pending->request = soup_session_request_uri (pending->thread_closure->session, (SoupURI*)(uri ? uri : next_mirror), error); + + /* Add caching headers. */ + if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_none_match != NULL) + { + glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request); + soup_message_headers_append (msg->request_headers, "If-None-Match", pending->if_none_match); + } + + if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_modified_since > 0) + { + glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request); + + g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since); + g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z"); + soup_message_headers_append (msg->request_headers, "If-Modified-Since", mod_date); + } } static void @@ -1050,7 +1073,14 @@ on_request_sent (GObject *object, if (SOUP_IS_REQUEST_HTTP (object)) { msg = soup_request_http_get_message ((SoupRequestHTTP*) object); - if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) + if (msg->status_code == SOUP_STATUS_NOT_MODIFIED && + (pending->if_none_match != NULL || pending->if_modified_since > 0)) + { + /* Version on the server is unchanged from the version we have cached locally; + * report this as an out-argument, a zero-length response buffer, and no error */ + pending->out_not_modified = TRUE; + } + else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code)) { /* is there another mirror we can try? */ if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len) @@ -1124,6 +1154,21 @@ on_request_sent (GObject *object, } goto out; } + + /* Grab cache properties from the response */ + pending->out_etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag")); + pending->out_last_modified = 0; + + const char *last_modified_str = soup_message_headers_get_one (msg->response_headers, "Last-Modified"); + if (last_modified_str != NULL) + { + SoupDate *soup_date = soup_date_new_from_string (last_modified_str); + if (soup_date != NULL) + { + pending->out_last_modified = soup_date_to_time_t (soup_date); + soup_date_free (soup_date); + } + } } pending->state = OSTREE_FETCHER_STATE_DOWNLOADING; @@ -1154,6 +1199,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, gboolean is_membuf, guint64 max_size, int priority, @@ -1175,6 +1222,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self, pending->mirrorlist = g_ptr_array_ref (mirrorlist); pending->filename = g_strdup (filename); pending->flags = flags; + pending->if_none_match = g_strdup (if_none_match); + pending->if_modified_since = if_modified_since; pending->max_size = max_size; pending->is_membuf = is_membuf; @@ -1196,13 +1245,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, FALSE, max_size, priority, cancellable, callback, user_data); } @@ -1211,6 +1263,9 @@ gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, GAsyncResult *result, GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error) { GTask *task; @@ -1231,6 +1286,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, *out_tmpf = pending->tmpf; pending->tmpf.initialized = FALSE; /* Transfer ownership */ + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + return TRUE; } @@ -1239,13 +1301,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, GAsyncReadyCallback callback, gpointer user_data) { - _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE, + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, + if_none_match, if_modified_since, TRUE, max_size, priority, cancellable, callback, user_data); } @@ -1254,6 +1319,9 @@ gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, GAsyncResult *result, GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error) { GTask *task; @@ -1274,6 +1342,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, g_assert (out_buf); *out_buf = ret; + if (out_not_modified != NULL) + *out_not_modified = pending->out_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&pending->out_etag); + if (out_last_modified != NULL) + *out_last_modified = pending->out_last_modified; + return TRUE; } diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index 1c3f8104..76f6bba1 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -34,6 +34,9 @@ typedef struct { GBytes *result_buf; + gboolean result_not_modified; + char *result_etag; + guint64 result_last_modified; /* second since the epoch */ gboolean done; GError **error; } @@ -48,6 +51,8 @@ fetch_uri_sync_on_complete (GObject *object, (void)_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)object, result, &data->result_buf, + &data->result_not_modified, + &data->result_etag, &data->result_last_modified, data->error); data->done = TRUE; } @@ -57,7 +62,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, GBytes **out_contents, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, guint64 max_size, GCancellable *cancellable, GError **error) @@ -79,7 +89,7 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe data.error = error; _ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename, - flags, + flags, if_none_match, if_modified_since, max_size, OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable, fetch_uri_sync_on_complete, &data); while (!data.done) @@ -94,6 +104,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe g_clear_error (error); ret = TRUE; *out_contents = NULL; + if (out_not_modified != NULL) + *out_not_modified = FALSE; + if (out_etag != NULL) + *out_etag = NULL; + if (out_last_modified != NULL) + *out_last_modified = 0; } } goto out; @@ -101,10 +117,17 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe ret = TRUE; *out_contents = g_steal_pointer (&data.result_buf); + if (out_not_modified != NULL) + *out_not_modified = data.result_not_modified; + if (out_etag != NULL) + *out_etag = g_steal_pointer (&data.result_etag); + if (out_last_modified != NULL) + *out_last_modified = data.result_last_modified; out: if (mainctx) g_main_context_pop_thread_default (mainctx); g_clear_pointer (&data.result_buf, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&data.result_etag, g_free); return ret; } @@ -113,8 +136,13 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint n_network_retries, GBytes **out_contents, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, guint64 max_size, GCancellable *cancellable, GError **error) @@ -127,7 +155,9 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher g_clear_error (&local_error); if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist, filename, flags, - out_contents, max_size, + if_none_match, if_modified_since, + out_contents, out_not_modified, out_etag, + out_last_modified, max_size, cancellable, &local_error)) return TRUE; } @@ -143,8 +173,13 @@ gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint n_network_retries, GBytes **out_contents, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, guint64 max_size, GCancellable *cancellable, GError **error) @@ -152,7 +187,9 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new (); g_ptr_array_add (mirrorlist, uri); /* no transfer */ return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags, - n_network_retries, out_contents, max_size, + if_none_match, if_modified_since, + n_network_retries, out_contents, + out_not_modified, out_etag, out_last_modified, max_size, cancellable, error); } diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h index 46935402..2cbaf5fa 100644 --- a/src/libostree/ostree-fetcher-util.h +++ b/src/libostree/ostree-fetcher-util.h @@ -56,8 +56,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint n_network_retries, GBytes **out_contents, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, guint64 max_size, GCancellable *cancellable, GError **error); @@ -65,8 +70,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher, gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher, OstreeFetcherURI *uri, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint n_network_retries, GBytes **out_contents, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, guint64 max_size, GCancellable *cancellable, GError **error); diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 32f3ea1b..2065ee54 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -123,6 +123,8 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, @@ -132,12 +134,17 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, GAsyncResult *result, GLnxTmpfile *out_tmpf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error); void _ostree_fetcher_request_to_membuf (OstreeFetcher *self, GPtrArray *mirrorlist, const char *filename, OstreeFetcherRequestFlags flags, + const char *if_none_match, + guint64 if_modified_since, guint64 max_size, int priority, GCancellable *cancellable, @@ -147,6 +154,9 @@ void _ostree_fetcher_request_to_membuf (OstreeFetcher *self, gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, GAsyncResult *result, GBytes **out_buf, + gboolean *out_not_modified, + char **out_etag, + guint64 *out_last_modified, GError **error); diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c index cb8a50e3..6381b31e 100644 --- a/src/libostree/ostree-metalink.c +++ b/src/libostree/ostree-metalink.c @@ -436,8 +436,10 @@ try_one_url (OstreeMetalinkRequest *self, if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher, uri, 0, + NULL, 0, self->metalink->n_network_retries, &bytes, + NULL, NULL, NULL, self->metalink->max_size, self->cancellable, error)) @@ -618,8 +620,9 @@ _ostree_metalink_request_sync (OstreeMetalink *self, request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL); if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0, + NULL, 0, self->n_network_retries, - &contents, self->max_size, + &contents, NULL, NULL, NULL, self->max_size, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f16ccec5..a0d983bb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -458,8 +458,9 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher, g_autoptr(GBytes) bytes = NULL; if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION, + NULL, 0, n_network_retries, - &bytes, + &bytes, NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) return FALSE; @@ -965,7 +966,7 @@ content_fetch_on_complete (GObject *object, OstreeObjectType objtype; gboolean free_fetch_data = TRUE; - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) goto out; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); @@ -1105,7 +1106,7 @@ meta_fetch_on_complete (GObject *object, g_debug ("fetch of %s%s complete", checksum_obj, fetch_data->is_detached_meta ? " (detached)" : ""); - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) { if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) { @@ -1282,7 +1283,7 @@ static_deltapart_fetch_on_complete (GObject *object, g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum); - if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error)) + if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error)) goto out; /* Transfer ownership of the fd */ @@ -1994,7 +1995,7 @@ start_fetch (OtPullData *pull_data, if (!is_meta && pull_data->trusted_http_direct) flags |= OSTREE_FETCHER_REQUEST_LINKABLE; _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist, - obj_subpath, flags, expected_max_size, + obj_subpath, flags, NULL, 0, expected_max_size, is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY : OSTREE_REPO_PULL_CONTENT_PRIORITY, pull_data->cancellable, @@ -2119,7 +2120,7 @@ start_fetch_deltapart (OtPullData *pull_data, g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, pull_data->content_mirrorlist, - deltapart_path, 0, fetch->size, + deltapart_path, 0, NULL, 0, fetch->size, OSTREE_FETCHER_DEFAULT_PRIORITY, pull_data->cancellable, static_deltapart_fetch_on_complete, @@ -2494,6 +2495,7 @@ on_superblock_fetched (GObject *src, if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, res, &delta_superblock_data, + NULL, NULL, NULL, error)) { if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) @@ -2570,6 +2572,7 @@ start_fetch_delta_superblock (OtPullData *pull_data, _ostree_fetcher_request_to_membuf (pull_data->fetcher, pull_data->content_mirrorlist, delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, OSTREE_MAX_METADATA_SIZE, 0, pull_data->cancellable, on_superblock_fetched, @@ -3000,8 +3003,10 @@ _ostree_preload_metadata_file (OstreeRepo *self, { return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, n_network_retries, - out_bytes, OSTREE_MAX_METADATA_SIZE, + out_bytes, NULL, NULL, NULL, + OSTREE_MAX_METADATA_SIZE, cancellable, error); } } @@ -3858,8 +3863,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, pull_data->n_network_retries, &bytes_sig, + NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -3887,8 +3894,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, pull_data->n_network_retries, &bytes_summary, + NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -3949,8 +3958,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, pull_data->n_network_retries, &bytes_summary, + NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -4012,8 +4023,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->meta_mirrorlist, "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, pull_data->n_network_retries, &bytes_summary, + NULL, NULL, NULL, OSTREE_MAX_METADATA_SIZE, cancellable, error)) goto out; @@ -5499,8 +5512,10 @@ find_remotes_cb (GObject *obj, mirrorlist, commit_filename, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, data->n_network_retries, &commit_bytes, + NULL, NULL, NULL, 0, /* no maximum size */ cancellable, &error))