packaging
/
ostree
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

libostree: Add support for ETag and Last-Modified headers 

Add support in the soup and curl fetchers to send the `If-None-Match`
and `If-Modified-Since` request headers, and pass on the `ETag` and
`Last-Modified` response headers.

This currently introduces no functional changes, but once call sites
provide the appropriate integration, this will allow HTTP caching to
happen with requests (typically with metadata requests, where the data
is not immutable due to being content-addressed). That should reduce
bandwidth requirements.

Signed-off-by: Philip Withnall <pwithnall@endlessos.org>
This commit is contained in:
Philip Withnall 2020-09-24 17:49:48 +01:00
parent 9f98aa9223
commit 407c683524
7 changed files with 410 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

251
src/libostree/ostree-fetcher-curl.c
View File

@ -99,10 +99,16 @@ struct FetcherRequest {
guint64 current_size;
guint64 max_size;
OstreeFetcherRequestFlags flags;
struct curl_slist *req_headers;
char *if_none_match; /* request ETag */
guint64 if_modified_since; /* seconds since the epoch */
gboolean is_membuf;
GError *caught_write_error;
GLnxTmpfile tmpf;
GString *output_buf;
gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */
char *out_etag; /* response ETag */
guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */
CURL *easy;
char error[CURL_ERROR_SIZE];
@ -335,7 +341,17 @@ check_multi_info (OstreeFetcher *fetcher)
else
{
curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);
if (!is_file && !(response >= 200 && response < 300))
if (!is_file && response == 304 &&
(req->if_none_match != NULL || req->if_modified_since > 0))
{
/* Version on the server is unchanged from the version we have
* cached locally; report this as an out-argument, a zero-length
* response buffer, and no error. */
req->out_not_modified = TRUE;
}
if (!is_file && !(response >= 200 && response < 300) && response != 304)
{
GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response);
@ -575,6 +591,175 @@ write_cb (void *ptr, size_t size, size_t nmemb, void *data)
return realsize;
}
/* @buf must already be known to be long enough */
static gboolean
parse_uint (const char *buf,
guint n_digits,
guint min,
guint max,
guint *out)
{
guint64 number;
const char *end_ptr = NULL;
gint saved_errno = 0;
g_return_val_if_fail (n_digits == 2 || n_digits == 4, FALSE);
g_return_val_if_fail (out != NULL, FALSE);
errno = 0;
number = g_ascii_strtoull (buf, (gchar **)&end_ptr, 10);
saved_errno = errno;
if (!g_ascii_isdigit (buf[0]) ||
saved_errno != 0 ||
end_ptr == NULL ||
end_ptr != buf + n_digits ||
number < min ||
number > max)
return FALSE;
*out = number;
return TRUE;
}
/* Locale-independent parsing for RFC 2616 date/times.
*
* Reference: https://tools.ietf.org/html/rfc2616#section-3.3.1
*
* Syntax:
* <day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT
*
* Note that this only accepts the full-year and GMT formats specified by
* RFC 1123. It doesn’t accept RFC 850 or asctime formats.
*
* Example:
* Wed, 21 Oct 2015 07:28:00 GMT
*/
static GDateTime *
parse_rfc2616_date_time (const char *buf,
size_t len)
{
guint day_int, year_int, hour_int, minute_int, second_int;
const char *day_names[] =
{
"Mon",
"Tue",
"Wed",
"Thu",
"Fri",
"Sat",
"Sun",
};
size_t day_name_index;
const char *month_names[] =
{
"Jan",
"Feb",
"Mar",
"Apr",
"May",
"Jun",
"Jul",
"Aug",
"Sep",
"Oct",
"Nov",
"Dec",
};
size_t month_name_index;
if (len != 29)
return NULL;
const char *day_name = buf;
const char *day = buf + 5;
const char *month_name = day + 3;
const char *year = month_name + 4;
const char *hour = year + 5;
const char *minute = hour + 3;
const char *second = minute + 3;
const char *tz = second + 3;
for (day_name_index = 0; day_name_index < G_N_ELEMENTS (day_names); day_name_index++)
{
if (strncmp (day_names[day_name_index], day_name, 3) == 0)
break;
}
if (day_name_index >= G_N_ELEMENTS (day_names))
return NULL;
/* don’t validate whether the day_name matches the rest of the date */
if (*(day_name + 3) != ',' || *(day_name + 4) != ' ')
return NULL;
if (!parse_uint (day, 2, 1, 31, &day_int))
return NULL;
if (*(day + 2) != ' ')
return NULL;
for (month_name_index = 0; month_name_index < G_N_ELEMENTS (month_names); month_name_index++)
{
if (strncmp (month_names[month_name_index], month_name, 3) == 0)
break;
}
if (month_name_index >= G_N_ELEMENTS (month_names))
return NULL;
if (*(month_name + 3) != ' ')
return NULL;
if (!parse_uint (year, 4, 0, 9999, &year_int))
return NULL;
if (*(year + 4) != ' ')
return NULL;
if (!parse_uint (hour, 2, 0, 23, &hour_int))
return NULL;
if (*(hour + 2) != ':')
return NULL;
if (!parse_uint (minute, 2, 0, 59, &minute_int))
return NULL;
if (*(minute + 2) != ':')
return NULL;
if (!parse_uint (second, 2, 0, 60, &second_int)) /* allow leap seconds */
return NULL;
if (*(second + 2) != ' ')
return NULL;
if (strncmp (tz, "GMT", 3) != 0)
return NULL;
return g_date_time_new_utc (year_int, month_name_index + 1, day_int,
hour_int, minute_int, second_int);
}
/* CURLOPT_HEADERFUNCTION */
static size_t
response_header_cb (const char *buffer, size_t size, size_t n_items, void *user_data)
{
const size_t real_size = size * n_items;
GTask *task = G_TASK (user_data);
FetcherRequest *req;
/* libcurl says that @size is always 1, but let’s check
* See https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION.html */
g_assert (size == 1);
req = g_task_get_task_data (task);
const char *etag_header = "ETag: ";
const char *last_modified_header = "Last-Modified: ";
if (real_size > strlen (etag_header) &&
strncasecmp (buffer, etag_header, strlen (etag_header)) == 0)
{
g_clear_pointer (&req->out_etag, g_free);
req->out_etag = g_strstrip (g_strdup (buffer + strlen (etag_header)));
}
else if (real_size > strlen (last_modified_header) &&
strncasecmp (buffer, last_modified_header, strlen (last_modified_header)) == 0)
{
g_autofree char *lm_buf = g_strstrip (g_strdup (buffer + strlen (last_modified_header)));
g_autoptr(GDateTime) dt = parse_rfc2616_date_time (lm_buf, strlen (lm_buf));
req->out_last_modified = (dt != NULL) ? g_date_time_to_unix (dt) : 0;
}
return real_size;
}
/* CURLOPT_PROGRESSFUNCTION */
static int
prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)
@ -600,6 +785,9 @@ request_unref (FetcherRequest *req)
glnx_tmpfile_clear (&req->tmpf);
if (req->output_buf)
g_string_free (req->output_buf, TRUE);
g_free (req->if_none_match);
g_free (req->out_etag);
g_clear_pointer (&req->req_headers, (GDestroyNotify)curl_slist_free_all);
curl_easy_cleanup (req->easy);
g_free (req);
@ -721,8 +909,29 @@ initiate_next_curl_request (FetcherRequest *req,
curl_easy_setopt (req->easy, CURLOPT_USERAGENT,
self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING);
if (self->extra_headers)
curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);
/* Set caching request headers */
if (req->if_none_match != NULL)
{
g_autofree char *if_none_match = g_strconcat ("If-None-Match: ", req->if_none_match, NULL);
req->req_headers = curl_slist_append (req->req_headers, if_none_match);
}
if (req->if_modified_since > 0)
{
g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (req->if_modified_since);
g_autofree char *mod_date = g_date_time_format (date_time, "If-Modified-Since: %a, %d %b %Y %H:%M:%S %Z");
req->req_headers = curl_slist_append (req->req_headers, mod_date);
}
/* Append a copy of @extra_headers to @req_headers, as the former could change
* between requests or while a request is in flight */
for (const struct curl_slist *l = self->extra_headers; l != NULL; l = l->next)
req->req_headers = curl_slist_append (req->req_headers, l->data);
if (req->req_headers != NULL)
curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, req->req_headers);
if (self->cookie_jar_path)
{
@ -796,6 +1005,7 @@ initiate_next_curl_request (FetcherRequest *req,
}
curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt (req->easy, CURLOPT_HEADERFUNCTION, response_header_cb);
if (g_getenv ("OSTREE_DEBUG_HTTP"))
curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);
curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error);
@ -815,6 +1025,7 @@ initiate_next_curl_request (FetcherRequest *req,
/* closure bindings -> task */
curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);
curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);
curl_easy_setopt (req->easy, CURLOPT_HEADERDATA, task);
curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);
CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy);
@ -826,6 +1037,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
gboolean is_membuf,
guint64 max_size,
int priority,
@ -859,6 +1072,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self,
req->filename = g_strdup (filename);
req->max_size = max_size;
req->flags = flags;
req->if_none_match = g_strdup (if_none_match);
req->if_modified_since = if_modified_since;
req->is_membuf = is_membuf;
/* We'll allocate the tmpfile on demand, so we handle
* file I/O errors just in the write func.
@ -882,13 +1097,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE,
_ostree_fetcher_request_async (self, mirrorlist, filename, flags,
if_none_match, if_modified_since, FALSE,
max_size, priority, cancellable,
callback, user_data);
}
@ -897,6 +1115,9 @@ gboolean
_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
GAsyncResult *result,
GLnxTmpfile *out_tmpf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error)
{
g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
@ -912,6 +1133,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
*out_tmpf = req->tmpf;
req->tmpf.initialized = FALSE; /* Transfer ownership */
if (out_not_modified != NULL)
*out_not_modified = req->out_not_modified;
if (out_etag != NULL)
*out_etag = g_steal_pointer (&req->out_etag);
if (out_last_modified != NULL)
*out_last_modified = req->out_last_modified;
return TRUE;
}
@ -920,13 +1148,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
_ostree_fetcher_request_async (self, mirrorlist, filename, flags,
if_none_match, if_modified_since, TRUE,
max_size, priority, cancellable,
callback, user_data);
}
@ -935,6 +1166,9 @@ gboolean
_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
GAsyncResult *result,
GBytes **out_buf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error)
{
GTask *task;
@ -955,6 +1189,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
g_assert (out_buf);
*out_buf = ret;
if (out_not_modified != NULL)
*out_not_modified = req->out_not_modified;
if (out_etag != NULL)
*out_etag = g_steal_pointer (&req->out_etag);
if (out_last_modified != NULL)
*out_last_modified = req->out_last_modified;
return TRUE;
}

81
src/libostree/ostree-fetcher-soup.c
View File

@ -90,9 +90,14 @@ typedef struct {
gboolean is_membuf;
OstreeFetcherRequestFlags flags;
char *if_none_match; /* request ETag */
guint64 if_modified_since; /* seconds since the epoch */
GInputStream *request_body;
GLnxTmpfile tmpf;
GOutputStream *out_stream;
gboolean out_not_modified; /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */
char *out_etag; /* response ETag */
guint64 out_last_modified; /* response Last-Modified, seconds since the epoch */
guint64 max_size;
guint64 current_size;
@ -196,8 +201,10 @@ pending_uri_unref (OstreeFetcherPendingURI *pending)
g_free (pending->filename);
g_clear_object (&pending->request);
g_clear_object (&pending->request_body);
g_free (pending->if_none_match);
glnx_tmpfile_clear (&pending->tmpf);
g_clear_object (&pending->out_stream);
g_free (pending->out_etag);
g_free (pending);
}
@ -431,6 +438,22 @@ create_pending_soup_request (OstreeFetcherPendingURI *pending,
pending->request = soup_session_request_uri (pending->thread_closure->session,
(SoupURI*)(uri ? uri : next_mirror), error);
/* Add caching headers. */
if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_none_match != NULL)
{
glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
soup_message_headers_append (msg->request_headers, "If-None-Match", pending->if_none_match);
}
if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_modified_since > 0)
{
glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since);
g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z");
soup_message_headers_append (msg->request_headers, "If-Modified-Since", mod_date);
}
}
static void
@ -1050,7 +1073,14 @@ on_request_sent (GObject *object,
if (SOUP_IS_REQUEST_HTTP (object))
{
msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
if (msg->status_code == SOUP_STATUS_NOT_MODIFIED &&
(pending->if_none_match != NULL || pending->if_modified_since > 0))
{
/* Version on the server is unchanged from the version we have cached locally;
* report this as an out-argument, a zero-length response buffer, and no error */
pending->out_not_modified = TRUE;
}
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
{
/* is there another mirror we can try? */
if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len)
@ -1124,6 +1154,21 @@ on_request_sent (GObject *object,
}
goto out;
}
/* Grab cache properties from the response */
pending->out_etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag"));
pending->out_last_modified = 0;
const char *last_modified_str = soup_message_headers_get_one (msg->response_headers, "Last-Modified");
if (last_modified_str != NULL)
{
SoupDate *soup_date = soup_date_new_from_string (last_modified_str);
if (soup_date != NULL)
{
pending->out_last_modified = soup_date_to_time_t (soup_date);
soup_date_free (soup_date);
}
}
}
pending->state = OSTREE_FETCHER_STATE_DOWNLOADING;
@ -1154,6 +1199,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
gboolean is_membuf,
guint64 max_size,
int priority,
@ -1175,6 +1222,8 @@ _ostree_fetcher_request_async (OstreeFetcher *self,
pending->mirrorlist = g_ptr_array_ref (mirrorlist);
pending->filename = g_strdup (filename);
pending->flags = flags;
pending->if_none_match = g_strdup (if_none_match);
pending->if_modified_since = if_modified_since;
pending->max_size = max_size;
pending->is_membuf = is_membuf;
@ -1196,13 +1245,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE,
_ostree_fetcher_request_async (self, mirrorlist, filename, flags,
if_none_match, if_modified_since, FALSE,
max_size, priority, cancellable,
callback, user_data);
}
@ -1211,6 +1263,9 @@ gboolean
_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
GAsyncResult *result,
GLnxTmpfile *out_tmpf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error)
{
GTask *task;
@ -1231,6 +1286,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
*out_tmpf = pending->tmpf;
pending->tmpf.initialized = FALSE; /* Transfer ownership */
if (out_not_modified != NULL)
*out_not_modified = pending->out_not_modified;
if (out_etag != NULL)
*out_etag = g_steal_pointer (&pending->out_etag);
if (out_last_modified != NULL)
*out_last_modified = pending->out_last_modified;
return TRUE;
}
@ -1239,13 +1301,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
_ostree_fetcher_request_async (self, mirrorlist, filename, flags,
if_none_match, if_modified_since, TRUE,
max_size, priority, cancellable,
callback, user_data);
}
@ -1254,6 +1319,9 @@ gboolean
_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
GAsyncResult *result,
GBytes **out_buf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error)
{
GTask *task;
@ -1274,6 +1342,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
g_assert (out_buf);
*out_buf = ret;
if (out_not_modified != NULL)
*out_not_modified = pending->out_not_modified;
if (out_etag != NULL)
*out_etag = g_steal_pointer (&pending->out_etag);
if (out_last_modified != NULL)
*out_last_modified = pending->out_last_modified;
return TRUE;
}

43
src/libostree/ostree-fetcher-util.c
View File

@ -34,6 +34,9 @@
typedef struct
{
GBytes *result_buf;
gboolean result_not_modified;
char *result_etag;
guint64 result_last_modified; /* second since the epoch */
gboolean done;
GError **error;
}
@ -48,6 +51,8 @@ fetch_uri_sync_on_complete (GObject *object,
(void)_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)object,
result, &data->result_buf,
&data->result_not_modified,
&data->result_etag, &data->result_last_modified,
data->error);
data->done = TRUE;
}
@ -57,7 +62,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
GBytes **out_contents,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
guint64 max_size,
GCancellable *cancellable,
GError **error)
@ -79,7 +89,7 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe
data.error = error;
_ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename,
flags,
flags, if_none_match, if_modified_since,
max_size, OSTREE_FETCHER_DEFAULT_PRIORITY,
cancellable, fetch_uri_sync_on_complete, &data);
while (!data.done)
@ -94,6 +104,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe
g_clear_error (error);
ret = TRUE;
*out_contents = NULL;
if (out_not_modified != NULL)
*out_not_modified = FALSE;
if (out_etag != NULL)
*out_etag = NULL;
if (out_last_modified != NULL)
*out_last_modified = 0;
}
}
goto out;
@ -101,10 +117,17 @@ _ostree_fetcher_mirrored_request_to_membuf_once (OstreeFetcher *fe
ret = TRUE;
*out_contents = g_steal_pointer (&data.result_buf);
if (out_not_modified != NULL)
*out_not_modified = data.result_not_modified;
if (out_etag != NULL)
*out_etag = g_steal_pointer (&data.result_etag);
if (out_last_modified != NULL)
*out_last_modified = data.result_last_modified;
out:
if (mainctx)
g_main_context_pop_thread_default (mainctx);
g_clear_pointer (&data.result_buf, (GDestroyNotify)g_bytes_unref);
g_clear_pointer (&data.result_etag, g_free);
return ret;
}
@ -113,8 +136,13 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint n_network_retries,
GBytes **out_contents,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
guint64 max_size,
GCancellable *cancellable,
GError **error)
@ -127,7 +155,9 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher
g_clear_error (&local_error);
if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist,
filename, flags,
out_contents, max_size,
if_none_match, if_modified_since,
out_contents, out_not_modified, out_etag,
out_last_modified, max_size,
cancellable, &local_error))
return TRUE;
}
@ -143,8 +173,13 @@ gboolean
_ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint n_network_retries,
GBytes **out_contents,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
guint64 max_size,
GCancellable *cancellable,
GError **error)
@ -152,7 +187,9 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
g_ptr_array_add (mirrorlist, uri); /* no transfer */
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags,
n_network_retries, out_contents, max_size,
if_none_match, if_modified_since,
n_network_retries, out_contents,
out_not_modified, out_etag, out_last_modified, max_size,
cancellable, error);
}

10
src/libostree/ostree-fetcher-util.h
View File

@ -56,8 +56,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint n_network_retries,
GBytes **out_contents,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
guint64 max_size,
GCancellable *cancellable,
GError **error);
@ -65,8 +70,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
OstreeFetcherURI *uri,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint n_network_retries,
GBytes **out_contents,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
guint64 max_size,
GCancellable *cancellable,
GError **error);

10
src/libostree/ostree-fetcher.h
View File

@ -123,6 +123,8 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
@ -132,12 +134,17 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self,
gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
GAsyncResult *result,
GLnxTmpfile *out_tmpf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error);
void _ostree_fetcher_request_to_membuf (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
const char *if_none_match,
guint64 if_modified_since,
guint64 max_size,
int priority,
GCancellable *cancellable,
@ -147,6 +154,9 @@ void _ostree_fetcher_request_to_membuf (OstreeFetcher *self,
gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
GAsyncResult *result,
GBytes **out_buf,
gboolean *out_not_modified,
char **out_etag,
guint64 *out_last_modified,
GError **error);

5
src/libostree/ostree-metalink.c
View File

@ -436,8 +436,10 @@ try_one_url (OstreeMetalinkRequest *self,
if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher,
uri, 0,
NULL, 0,
self->metalink->n_network_retries,
&bytes,
NULL, NULL, NULL,
self->metalink->max_size,
self->cancellable,
error))
@ -618,8 +620,9 @@ _ostree_metalink_request_sync (OstreeMetalink *self,
request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL);
if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0,
NULL, 0,
self->n_network_retries,
&contents, self->max_size,
&contents, NULL, NULL, NULL, self->max_size,
cancellable, error))
goto out;

29
src/libostree/ostree-repo-pull.c
View File

@ -458,8 +458,9 @@ fetch_mirrored_uri_contents_utf8_sync (OstreeFetcher *fetcher,
g_autoptr(GBytes) bytes = NULL;
if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist,
filename, OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
NULL, 0,
n_network_retries,
&bytes,
&bytes, NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
return FALSE;
@ -965,7 +966,7 @@ content_fetch_on_complete (GObject *object,
OstreeObjectType objtype;
gboolean free_fetch_data = TRUE;
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
goto out;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@ -1105,7 +1106,7 @@ meta_fetch_on_complete (GObject *object,
g_debug ("fetch of %s%s complete", checksum_obj,
fetch_data->is_detached_meta ? " (detached)" : "");
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
{
if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
{
@ -1282,7 +1283,7 @@ static_deltapart_fetch_on_complete (GObject *object,
g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, NULL, NULL, NULL, error))
goto out;
/* Transfer ownership of the fd */
@ -1994,7 +1995,7 @@ start_fetch (OtPullData *pull_data,
if (!is_meta && pull_data->trusted_http_direct)
flags |= OSTREE_FETCHER_REQUEST_LINKABLE;
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
obj_subpath, flags, expected_max_size,
obj_subpath, flags, NULL, 0, expected_max_size,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
@ -2119,7 +2120,7 @@ start_fetch_deltapart (OtPullData *pull_data,
g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, 0, fetch->size,
deltapart_path, 0, NULL, 0, fetch->size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
@ -2494,6 +2495,7 @@ on_superblock_fetched (GObject *src,
if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
res,
&delta_superblock_data,
NULL, NULL, NULL,
error))
{
if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@ -2570,6 +2572,7 @@ start_fetch_delta_superblock (OtPullData *pull_data,
_ostree_fetcher_request_to_membuf (pull_data->fetcher,
pull_data->content_mirrorlist,
delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
OSTREE_MAX_METADATA_SIZE,
0, pull_data->cancellable,
on_superblock_fetched,
@ -3000,8 +3003,10 @@ _ostree_preload_metadata_file (OstreeRepo *self,
{
return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
n_network_retries,
out_bytes, OSTREE_MAX_METADATA_SIZE,
out_bytes, NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error);
}
}
@ -3858,8 +3863,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
pull_data->n_network_retries,
&bytes_sig,
NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@ -3887,8 +3894,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
pull_data->meta_mirrorlist,
"summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
pull_data->n_network_retries,
&bytes_summary,
NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@ -3949,8 +3958,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
pull_data->n_network_retries,
&bytes_summary,
NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@ -4012,8 +4023,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->meta_mirrorlist,
"summary",
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
pull_data->n_network_retries,
&bytes_summary,
NULL, NULL, NULL,
OSTREE_MAX_METADATA_SIZE,
cancellable, error))
goto out;
@ -5499,8 +5512,10 @@ find_remotes_cb (GObject *obj,
mirrorlist,
commit_filename,
OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
NULL, 0,
data->n_network_retries,
&commit_bytes,
NULL, NULL, NULL,
0, /* no maximum size */
cancellable,
&error))