libostree: Add support for ETag and Last-Modified headers

Add support in the soup and curl fetchers to send the `If-None-Match`
and `If-Modified-Since` request headers, and pass on the `ETag` and
`Last-Modified` response headers.

This currently introduces no functional changes, but once call sites
provide the appropriate integration, this will allow HTTP caching to
happen with requests (typically with metadata requests, where the data
is not immutable due to being content-addressed). That should reduce
bandwidth requirements.

Signed-off-by: Philip Withnall <pwithnall@endlessos.org>

src/libostree/ostree-fetcher-curl.c

@@ -99,10 +99,16 @@ struct FetcherRequest {
   guint64 current_size;
   guint64 max_size;
   OstreeFetcherRequestFlags flags;
+  struct curl_slist *req_headers;
+  char *if_none_match;  /* request ETag */
+  guint64 if_modified_since;  /* seconds since the epoch */
   gboolean is_membuf;
   GError *caught_write_error;
   GLnxTmpfile tmpf;
   GString *output_buf;
+  gboolean out_not_modified;  /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */
+  char *out_etag;  /* response ETag */
+  guint64 out_last_modified;  /* response Last-Modified, seconds since the epoch */
 
   CURL *easy;
   char error[CURL_ERROR_SIZE];
@@ -335,7 +341,17 @@ check_multi_info (OstreeFetcher *fetcher)
       else
         {
           curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);
-          if (!is_file && !(response >= 200 && response < 300))
+
+          if (!is_file && response == 304 &&
+              (req->if_none_match != NULL || req->if_modified_since > 0))
+            {
+              /* Version on the server is unchanged from the version we have
+               * cached locally; report this as an out-argument, a zero-length
+               * response buffer, and no error. */
+              req->out_not_modified = TRUE;
+            }
+
+          if (!is_file && !(response >= 200 && response < 300) && response != 304)
             {
               GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response);
 
@@ -575,6 +591,175 @@ write_cb (void *ptr, size_t size, size_t nmemb, void *data)
   return realsize;
 }
 
+/* @buf must already be known to be long enough */
+static gboolean
+parse_uint (const char *buf,
+            guint       n_digits,
+            guint       min,
+            guint       max,
+            guint      *out)
+{
+  guint64 number;
+  const char *end_ptr = NULL;
+  gint saved_errno = 0;
+
+  g_return_val_if_fail (n_digits == 2 || n_digits == 4, FALSE);
+  g_return_val_if_fail (out != NULL, FALSE);
+
+  errno = 0;
+  number = g_ascii_strtoull (buf, (gchar **)&end_ptr, 10);
+  saved_errno = errno;
+
+  if (!g_ascii_isdigit (buf[0]) ||
+      saved_errno != 0 ||
+      end_ptr == NULL ||
+      end_ptr != buf + n_digits ||
+      number < min ||
+      number > max)
+    return FALSE;
+
+  *out = number;
+  return TRUE;
+}
+
+/* Locale-independent parsing for RFC 2616 date/times.
+ *
+ * Reference: https://tools.ietf.org/html/rfc2616#section-3.3.1
+ *
+ * Syntax:
+ *    <day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT
+ *
+ * Note that this only accepts the full-year and GMT formats specified by
+ * RFC 1123. It doesn’t accept RFC 850 or asctime formats.
+ *
+ * Example:
+ *    Wed, 21 Oct 2015 07:28:00 GMT
+ */
+static GDateTime *
+parse_rfc2616_date_time (const char *buf,
+                         size_t      len)
+{
+  guint day_int, year_int, hour_int, minute_int, second_int;
+  const char *day_names[] =
+    {
+      "Mon",
+      "Tue",
+      "Wed",
+      "Thu",
+      "Fri",
+      "Sat",
+      "Sun",
+    };
+  size_t day_name_index;
+  const char *month_names[] =
+    {
+      "Jan",
+      "Feb",
+      "Mar",
+      "Apr",
+      "May",
+      "Jun",
+      "Jul",
+      "Aug",
+      "Sep",
+      "Oct",
+      "Nov",
+      "Dec",
+    };
+  size_t month_name_index;
+
+  if (len != 29)
+    return NULL;
+
+  const char *day_name = buf;
+  const char *day = buf + 5;
+  const char *month_name = day + 3;
+  const char *year = month_name + 4;
+  const char *hour = year + 5;
+  const char *minute = hour + 3;
+  const char *second = minute + 3;
+  const char *tz = second + 3;
+
+  for (day_name_index = 0; day_name_index < G_N_ELEMENTS (day_names); day_name_index++)
+    {
+      if (strncmp (day_names[day_name_index], day_name, 3) == 0)
+        break;
+    }
+  if (day_name_index >= G_N_ELEMENTS (day_names))
+    return NULL;
+  /* don’t validate whether the day_name matches the rest of the date */
+  if (*(day_name + 3) != ',' || *(day_name + 4) != ' ')
+    return NULL;
+  if (!parse_uint (day, 2, 1, 31, &day_int))
+    return NULL;
+  if (*(day + 2) != ' ')
+    return NULL;
+  for (month_name_index = 0; month_name_index < G_N_ELEMENTS (month_names); month_name_index++)
+    {
+      if (strncmp (month_names[month_name_index], month_name, 3) == 0)
+        break;
+    }
+  if (month_name_index >= G_N_ELEMENTS (month_names))
+    return NULL;
+  if (*(month_name + 3) != ' ')
+    return NULL;
+  if (!parse_uint (year, 4, 0, 9999, &year_int))
+    return NULL;
+  if (*(year + 4) != ' ')
+    return NULL;
+  if (!parse_uint (hour, 2, 0, 23, &hour_int))
+    return NULL;
+  if (*(hour + 2) != ':')
+    return NULL;
+  if (!parse_uint (minute, 2, 0, 59, &minute_int))
+    return NULL;
+  if (*(minute + 2) != ':')
+    return NULL;
+  if (!parse_uint (second, 2, 0, 60, &second_int))  /* allow leap seconds */
+    return NULL;
+  if (*(second + 2) != ' ')
+    return NULL;
+  if (strncmp (tz, "GMT", 3) != 0)
+    return NULL;
+
+  return g_date_time_new_utc (year_int, month_name_index + 1, day_int,
+                              hour_int, minute_int, second_int);
+}
+
+/* CURLOPT_HEADERFUNCTION */
+static size_t
+response_header_cb (const char *buffer, size_t size, size_t n_items, void *user_data)
+{
+  const size_t real_size = size * n_items;
+  GTask *task = G_TASK (user_data);
+  FetcherRequest *req;
+
+  /* libcurl says that @size is always 1, but let’s check
+   * See https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION.html */
+  g_assert (size == 1);
+
+  req = g_task_get_task_data (task);
+
+  const char *etag_header = "ETag: ";
+  const char *last_modified_header = "Last-Modified: ";
+
+  if (real_size > strlen (etag_header) &&
+      strncasecmp (buffer, etag_header, strlen (etag_header)) == 0)
+    {
+      g_clear_pointer (&req->out_etag, g_free);
+      req->out_etag = g_strstrip (g_strdup (buffer + strlen (etag_header)));
+    }
+  else if (real_size > strlen (last_modified_header) &&
+           strncasecmp (buffer, last_modified_header, strlen (last_modified_header)) == 0)
+    {
+      g_autofree char *lm_buf = g_strstrip (g_strdup (buffer + strlen (last_modified_header)));
+      g_autoptr(GDateTime) dt = parse_rfc2616_date_time (lm_buf, strlen (lm_buf));
+      req->out_last_modified = (dt != NULL) ? g_date_time_to_unix (dt) : 0;
+    }
+
+  return real_size;
+}
+
 /* CURLOPT_PROGRESSFUNCTION */
 static int
 prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)
@@ -600,6 +785,9 @@ request_unref (FetcherRequest *req)
   glnx_tmpfile_clear (&req->tmpf);
   if (req->output_buf)
     g_string_free (req->output_buf, TRUE);
+  g_free (req->if_none_match);
+  g_free (req->out_etag);
+  g_clear_pointer (&req->req_headers, (GDestroyNotify)curl_slist_free_all);
   curl_easy_cleanup (req->easy);
 
   g_free (req);
@@ -721,8 +909,29 @@ initiate_next_curl_request (FetcherRequest *req,
 
   curl_easy_setopt (req->easy, CURLOPT_USERAGENT,
                     self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING);
-  if (self->extra_headers)
-    curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);
+
+  /* Set caching request headers */
+  if (req->if_none_match != NULL)
+    {
+      g_autofree char *if_none_match = g_strconcat ("If-None-Match: ", req->if_none_match, NULL);
+      req->req_headers = curl_slist_append (req->req_headers, if_none_match);
+    }
+
+  if (req->if_modified_since > 0)
+    {
+      g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (req->if_modified_since);
+      g_autofree char *mod_date = g_date_time_format (date_time, "If-Modified-Since: %a, %d %b %Y %H:%M:%S %Z");
+
+      req->req_headers = curl_slist_append (req->req_headers, mod_date);
+    }
+
+  /* Append a copy of @extra_headers to @req_headers, as the former could change
+   * between requests or while a request is in flight */
+  for (const struct curl_slist *l = self->extra_headers; l != NULL; l = l->next)
+    req->req_headers = curl_slist_append (req->req_headers, l->data);
+
+  if (req->req_headers != NULL)
+    curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, req->req_headers);
 
   if (self->cookie_jar_path)
     {
@@ -796,6 +1005,7 @@ initiate_next_curl_request (FetcherRequest *req,
     }
 
   curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
+  curl_easy_setopt (req->easy, CURLOPT_HEADERFUNCTION, response_header_cb);
   if (g_getenv ("OSTREE_DEBUG_HTTP"))
     curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);
   curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error);
@@ -815,6 +1025,7 @@ initiate_next_curl_request (FetcherRequest *req,
   /* closure bindings -> task */
   curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);
   curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);
+  curl_easy_setopt (req->easy, CURLOPT_HEADERDATA, task);
   curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);
 
   CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy);
@@ -826,6 +1037,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
                                GPtrArray             *mirrorlist,
                                const char            *filename,
                                OstreeFetcherRequestFlags flags,
+                               const char            *if_none_match,
+                               guint64                if_modified_since,
                                gboolean               is_membuf,
                                guint64                max_size,
                                int                    priority,
@@ -859,6 +1072,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
   req->filename = g_strdup (filename);
   req->max_size = max_size;
   req->flags = flags;
+  req->if_none_match = g_strdup (if_none_match);
+  req->if_modified_since = if_modified_since;
   req->is_membuf = is_membuf;
   /* We'll allocate the tmpfile on demand, so we handle
    * file I/O errors just in the write func.
@@ -882,13 +1097,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
                                     GPtrArray             *mirrorlist,
                                     const char            *filename,
                                     OstreeFetcherRequestFlags flags,
+                                    const char            *if_none_match,
+                                    guint64                if_modified_since,
                                     guint64                max_size,
                                     int                    priority,
                                     GCancellable          *cancellable,
                                     GAsyncReadyCallback    callback,
                                     gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, FALSE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -897,6 +1115,9 @@ gboolean
 _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                            GAsyncResult  *result,
                                            GLnxTmpfile   *out_tmpf,
+                                           gboolean      *out_not_modified,
+                                           char         **out_etag,
+                                           guint64       *out_last_modified,
                                            GError       **error)
 {
   g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
@@ -912,6 +1133,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
   *out_tmpf = req->tmpf;
   req->tmpf.initialized = FALSE; /* Transfer ownership */
 
+  if (out_not_modified != NULL)
+    *out_not_modified = req->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&req->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = req->out_last_modified;
+
   return TRUE;
 }
 
@@ -920,13 +1148,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
                                    GPtrArray             *mirrorlist,
                                    const char            *filename,
                                    OstreeFetcherRequestFlags flags,
+                                   const char            *if_none_match,
+                                   guint64                if_modified_since,
                                    guint64                max_size,
                                    int                    priority,
                                    GCancellable          *cancellable,
                                    GAsyncReadyCallback    callback,
                                    gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, TRUE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -935,6 +1166,9 @@ gboolean
 _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                           GAsyncResult  *result,
                                           GBytes       **out_buf,
+                                          gboolean      *out_not_modified,
+                                          char         **out_etag,
+                                          guint64       *out_last_modified,
                                           GError       **error)
 {
   GTask *task;
@@ -955,6 +1189,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
   g_assert (out_buf);
   *out_buf = ret;
 
+  if (out_not_modified != NULL)
+    *out_not_modified = req->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&req->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = req->out_last_modified;
+
   return TRUE;
 }