New upstream version 2020.8

src/libostree/libostree-devel.sym

@@ -15,7 +15,12 @@
   License along with this library; if not, write to the
   Free Software Foundation, Inc., 59 Temple Place - Suite 330,
   Boston, MA 02111-1307, USA.
-***/
+
+  How to introduce the first new symbol for a development version:
+   - copy the stub content below to a new entry
+   - replace templated value as noted
+   - uncomment the include in Makefile-libostree.am
+*/
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste

src/libostree/libostree-released.sym

@@ -596,7 +596,6 @@ global:
 /* No new symbols in 2020.2 */
 /* No new symbols in 2020.3 */
 
-/* Add new symbols here.  Release commits should copy this section into -released.sym. */
 LIBOSTREE_2020.4 {
 global:
   ostree_repo_commit_modifier_set_sepolicy_from_commit;
@@ -619,14 +618,10 @@ global:
 } LIBOSTREE_2020.1;
 
 /* No new symbols in 2020.5 */
-
 /* No new symbols in 2020.6 */
 
 LIBOSTREE_2020.7 {
 global:
-  /* Add symbols here, and uncomment the bits in
-   * Makefile-libostree.am to enable this too.
-   */
   ostree_repo_static_delta_execute_offline_with_signature;
   ostree_repo_static_delta_verify_signature;
   ostree_bootconfig_parser_get_overlay_initrds;
@@ -636,6 +631,13 @@ global:
   ostree_sysroot_stage_overlay_initrd;
 } LIBOSTREE_2020.4;
 
+LIBOSTREE_2020.8 {
+global:
+  ostree_repo_list_static_delta_indexes;
+  ostree_repo_static_delta_reindex;
+  ostree_repo_gpg_sign_data;
+} LIBOSTREE_2020.7;
+
 /* NOTE: Only add more content here in release commits!  See the
  * comments at the top of this file.
  */

src/libostree/ostree-core-private.h

@@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
                                              const char        *to,
                                              guint              i);
 
+char *
+_ostree_get_relative_static_delta_index_path (const char        *to);
+
 static inline char * _ostree_get_commitpartial_path (const char *checksum)
 {
   return g_strconcat ("state/", checksum, ".commitpartial", NULL);

src/libostree/ostree-core.c

@@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char         *checksum,
   return g_string_free (path, FALSE);
 }
 
-char *
-_ostree_get_relative_static_delta_path (const char *from,
-                                        const char *to,
-                                        const char *target)
+static GString *
+static_delta_path_base (const char *dir,
+                        const char *from,
+                        const char *to)
 {
   guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
   char to_b64[44];
   guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
-  GString *ret = g_string_new ("deltas/");
+  GString *ret = g_string_new (dir);
 
   ostree_checksum_inplace_to_bytes (to, csum_to);
   ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
@@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from,
     g_string_append_c (ret, '/');
   g_string_append (ret, to_b64 + 2);
 
+  return ret;
+}
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+                                        const char *to,
+                                        const char *target)
+{
+  GString *ret = static_delta_path_base ("deltas/", from, to);
+
   if (target != NULL)
     {
       g_string_append_c (ret, '/');
@@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
   return _ostree_get_relative_static_delta_path (from, to, partstr);
 }
 
+char *
+_ostree_get_relative_static_delta_index_path (const char *to)
+{
+  GString *ret = static_delta_path_base ("delta-indexes/", NULL, to);
+
+  g_string_append (ret, ".index");
+
+  return g_string_free (ret, FALSE);
+}
+
 gboolean
 _ostree_parse_delta_name (const char   *delta_name,
                           char        **out_from,

src/libostree/ostree-date-utils-private.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2020 Endless OS Foundation LLC
+ *
+ * SPDX-License-Identifier: LGPL-2.0+
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Authors:
+ *  - Philip Withnall <pwithnall@endlessos.org>
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include <glib.h>
+
+G_BEGIN_DECLS
+
+GDateTime *_ostree_parse_rfc2616_date_time (const char *buf,
+                                            size_t      len);
+
+G_END_DECLS
+
+#endif

src/libostree/ostree-date-utils.c

@@ -0,0 +1,166 @@
+/*
+ * Copyright © 2020 Endless OS Foundation LLC
+ *
+ * SPDX-License-Identifier: LGPL-2.0+
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Authors:
+ *  - Philip Withnall <pwithnall@endlessos.org>
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <glib.h>
+#include <string.h>
+
+#include "ostree-date-utils-private.h"
+
+/* @buf must already be known to be long enough */
+static gboolean
+parse_uint (const char *buf,
+            guint       n_digits,
+            guint       min,
+            guint       max,
+            guint      *out)
+{
+  guint64 number;
+  const char *end_ptr = NULL;
+  gint saved_errno = 0;
+
+  g_return_val_if_fail (n_digits == 2 || n_digits == 4, FALSE);
+  g_return_val_if_fail (out != NULL, FALSE);
+
+  errno = 0;
+  number = g_ascii_strtoull (buf, (gchar **)&end_ptr, 10);
+  saved_errno = errno;
+
+  if (!g_ascii_isdigit (buf[0]) ||
+      saved_errno != 0 ||
+      end_ptr == NULL ||
+      end_ptr != buf + n_digits ||
+      number < min ||
+      number > max)
+    return FALSE;
+
+  *out = number;
+  return TRUE;
+}
+
+/* Locale-independent parsing for RFC 2616 date/times.
+ *
+ * Reference: https://tools.ietf.org/html/rfc2616#section-3.3.1
+ *
+ * Syntax:
+ *    <day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT
+ *
+ * Note that this only accepts the full-year and GMT formats specified by
+ * RFC 1123. It doesn’t accept RFC 850 or asctime formats.
+ *
+ * Example:
+ *    Wed, 21 Oct 2015 07:28:00 GMT
+ */
+GDateTime *
+_ostree_parse_rfc2616_date_time (const char *buf,
+                                 size_t      len)
+{
+  guint day_int, year_int, hour_int, minute_int, second_int;
+  const char *day_names[] =
+    {
+      "Mon",
+      "Tue",
+      "Wed",
+      "Thu",
+      "Fri",
+      "Sat",
+      "Sun",
+    };
+  size_t day_name_index;
+  const char *month_names[] =
+    {
+      "Jan",
+      "Feb",
+      "Mar",
+      "Apr",
+      "May",
+      "Jun",
+      "Jul",
+      "Aug",
+      "Sep",
+      "Oct",
+      "Nov",
+      "Dec",
+    };
+  size_t month_name_index;
+
+  if (len != 29)
+    return NULL;
+
+  const char *day_name = buf;
+  const char *day = buf + 5;
+  const char *month_name = day + 3;
+  const char *year = month_name + 4;
+  const char *hour = year + 5;
+  const char *minute = hour + 3;
+  const char *second = minute + 3;
+  const char *tz = second + 3;
+
+  for (day_name_index = 0; day_name_index < G_N_ELEMENTS (day_names); day_name_index++)
+    {
+      if (strncmp (day_names[day_name_index], day_name, 3) == 0)
+        break;
+    }
+  if (day_name_index >= G_N_ELEMENTS (day_names))
+    return NULL;
+  /* don’t validate whether the day_name matches the rest of the date */
+  if (*(day_name + 3) != ',' || *(day_name + 4) != ' ')
+    return NULL;
+  if (!parse_uint (day, 2, 1, 31, &day_int))
+    return NULL;
+  if (*(day + 2) != ' ')
+    return NULL;
+  for (month_name_index = 0; month_name_index < G_N_ELEMENTS (month_names); month_name_index++)
+    {
+      if (strncmp (month_names[month_name_index], month_name, 3) == 0)
+        break;
+    }
+  if (month_name_index >= G_N_ELEMENTS (month_names))
+    return NULL;
+  if (*(month_name + 3) != ' ')
+    return NULL;
+  if (!parse_uint (year, 4, 0, 9999, &year_int))
+    return NULL;
+  if (*(year + 4) != ' ')
+    return NULL;
+  if (!parse_uint (hour, 2, 0, 23, &hour_int))
+    return NULL;
+  if (*(hour + 2) != ':')
+    return NULL;
+  if (!parse_uint (minute, 2, 0, 59, &minute_int))
+    return NULL;
+  if (*(minute + 2) != ':')
+    return NULL;
+  if (!parse_uint (second, 2, 0, 60, &second_int))  /* allow leap seconds */
+    return NULL;
+  if (*(second + 2) != ' ')
+    return NULL;
+  if (strncmp (tz, "GMT", 3) != 0)
+    return NULL;
+
+  return g_date_time_new_utc (year_int, month_name_index + 1, day_int,
+                              hour_int, minute_int, second_int);
+}

src/libostree/ostree-deployment.c

@@ -27,12 +27,24 @@ typedef GObjectClass OstreeDeploymentClass;
 
 G_DEFINE_TYPE (OstreeDeployment, ostree_deployment, G_TYPE_OBJECT)
 
+/*
+ * ostree_deployment_get_csum:
+ * @self: Deployment
+ *
+ * Returns: (not nullable): The OSTree commit used for this deployment.
+ */
 const char *
 ostree_deployment_get_csum (OstreeDeployment *self)
 {
   return self->csum;
 }
 
+/*
+ * ostree_deployment_get_bootcsum:
+ * @self: Deployment
+ *
+ * Returns: (not nullable): The "boot checksum" for content installed in `/boot/ostree`.
+ */
 const char *
 ostree_deployment_get_bootcsum (OstreeDeployment *self)
 {
@@ -41,9 +53,9 @@ ostree_deployment_get_bootcsum (OstreeDeployment *self)
 
 /*
  * ostree_deployment_get_osname:
- * @self: Deployemnt
+ * @self: Deployment
  *
- * Returns: The "stateroot" name, also known as an "osname"
+ * Returns: (not nullable): The "stateroot" name, also known as an "osname"
  */
 const char *
 ostree_deployment_get_osname (OstreeDeployment *self)
@@ -51,12 +63,24 @@ ostree_deployment_get_osname (OstreeDeployment *self)
   return self->osname;
 }
 
+/*
+ * ostree_deployment_get_deployserial:
+ * @self: Deployment
+ *
+ * Returns: An integer counter used to ensure multiple deployments of a commit are unique
+ */
 int
 ostree_deployment_get_deployserial (OstreeDeployment *self)
 {
   return self->deployserial;
 }
 
+/*
+ * ostree_deployment_get_bootserial:
+ * @self: Deployment
+ *
+ * Returns: An integer counter to index from shared kernels into deployments
+ */
 int
 ostree_deployment_get_bootserial (OstreeDeployment *self)
 {
@@ -87,24 +111,51 @@ ostree_deployment_get_origin (OstreeDeployment *self)
   return self->origin;
 }
 
+/**
+ * ostree_deployment_get_index:
+ * @self: Deployment
+ *
+ * Returns: The global index into the bootloader ordering
+ */
 int
 ostree_deployment_get_index (OstreeDeployment *self)
 {
   return self->index;
 }
 
+/**
+ * ostree_deployment_set_index:
+ * @self: Deployment
+ * @index: Index into bootloader ordering
+ *
+ * Sets the global index into the bootloader ordering.
+ */
 void
 ostree_deployment_set_index (OstreeDeployment *self, int index)
 {
   self->index = index;
 }
 
+/**
+ * ostree_deployment_set_bootserial:
+ * @self: Deployment
+ * @index: Don't use this
+ *
+ * Should never have been made public API; don't use this.
+ */
 void
 ostree_deployment_set_bootserial (OstreeDeployment *self, int index)
 {
   self->bootserial = index;
 }
 
+/**
+ * ostree_deployment_set_bootconfig:
+ * @self: Deployment
+ * @bootconfig: (nullable): Bootloader configuration object
+ *
+ * Set or clear the bootloader configuration.
+ */
 void
 ostree_deployment_set_bootconfig (OstreeDeployment *self, OstreeBootconfigParser *bootconfig)
 {
@@ -113,6 +164,14 @@ ostree_deployment_set_bootconfig (OstreeDeployment *self, OstreeBootconfigParser
     self->bootconfig = g_object_ref (bootconfig);
 }
 
+/**
+ * ostree_deployment_set_origin:
+ * @self: Deployment
+ * @origin: (nullable): Set the origin for this deployment
+ *
+ * Replace the "origin", which is a description of the source
+ * of the deployment and how to update to the next version.
+ */
 void
 ostree_deployment_set_origin (OstreeDeployment *self, GKeyFile *origin)
 {
@@ -190,7 +249,7 @@ _ostree_deployment_get_overlay_initrds (OstreeDeployment *self)
  * ostree_deployment_clone:
  * @self: Deployment
  *
- * Returns: (transfer full): New deep copy of @self
+ * Returns: (not nullable) (transfer full): New deep copy of @self
  */
 OstreeDeployment *
 ostree_deployment_clone (OstreeDeployment *self)
@@ -224,6 +283,12 @@ ostree_deployment_clone (OstreeDeployment *self)
   return ret;
 }
 
+/**
+ * ostree_deployment_hash:
+ * @v: (type OstreeDeployment): Deployment
+ *
+ * Returns: An integer suitable for use in a `GHashTable`
+ */
 guint
 ostree_deployment_hash (gconstpointer v)
 {
@@ -287,6 +352,17 @@ ostree_deployment_class_init (OstreeDeploymentClass *class)
   object_class->finalize = ostree_deployment_finalize;
 }
 
+/**
+ * ostree_deployment_new:
+ * @index: Global index into the bootloader entries
+ * @osname: "stateroot" for this deployment
+ * @csum: OSTree commit that will be deployed
+ * @deployserial: Unique counter
+ * @bootcsum: (nullable): Kernel/initrd checksum
+ * @bootserial: Unique index
+ *
+ * Returns: (transfer full) (not nullable): New deployment
+ */
 OstreeDeployment *
 ostree_deployment_new (int    index,
                    const char  *osname,
@@ -323,7 +399,7 @@ ostree_deployment_new (int    index,
  * access, it, you must either use fd-relative api such as openat(),
  * or concatenate it with the full ostree_sysroot_get_path().
  *
- * Returns: (transfer full): Path to deployment root directory, relative to sysroot
+ * Returns: (not nullable) (transfer full): Path to deployment root directory, relative to sysroot
  */
 char *
 ostree_deployment_get_origin_relpath (OstreeDeployment *self)
@@ -337,6 +413,7 @@ ostree_deployment_get_origin_relpath (OstreeDeployment *self)
 /**
  * ostree_deployment_unlocked_state_to_string:
  *
+ * Returns: (not nullable): Description of state
  * Since: 2016.4
  */
 const char *

src/libostree/ostree-diff.h

@@ -42,7 +42,7 @@ typedef enum {
 typedef struct _OstreeDiffItem OstreeDiffItem;
 struct _OstreeDiffItem
 {
-  volatile gint refcount;
+  gint refcount;  /* atomic */
 
   GFile *src;
   GFile *target;

src/libostree/ostree-enumtypes.c.template

@@ -36,9 +36,9 @@
 GType
 _@enum_name@_get_type (void)
 {
-  static volatile gsize the_type__volatile = 0;
+  static gsize static_the_type = 0;
 
-  if (g_once_init_enter (&the_type__volatile))
+  if (g_once_init_enter (&static_the_type))
     {
       static const G@Type@Value values[] = {
 /*** END value-header ***/
@@ -57,10 +57,10 @@ _@enum_name@_get_type (void)
         g_intern_static_string ("@EnumName@"),
         values);
 
-      g_once_init_leave (&the_type__volatile, the_type);
+      g_once_init_leave (&static_the_type, the_type);
     }
 
-  return the_type__volatile;
+  return static_the_type;
 }
 
 /*** END value-tail ***/

src/libostree/ostree-fetcher-curl.c

@@ -45,6 +45,7 @@
 #define CURLPIPE_MULTIPLEX 0
 #endif
 
+#include "ostree-date-utils-private.h"
 #include "ostree-fetcher.h"
 #include "ostree-fetcher-util.h"
 #include "ostree-enumtypes.h"
@@ -99,10 +100,16 @@ struct FetcherRequest {
   guint64 current_size;
   guint64 max_size;
   OstreeFetcherRequestFlags flags;
+  struct curl_slist *req_headers;
+  char *if_none_match;  /* request ETag */
+  guint64 if_modified_since;  /* seconds since the epoch */
   gboolean is_membuf;
   GError *caught_write_error;
   GLnxTmpfile tmpf;
   GString *output_buf;
+  gboolean out_not_modified;  /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */
+  char *out_etag;  /* response ETag */
+  guint64 out_last_modified;  /* response Last-Modified, seconds since the epoch */
 
   CURL *easy;
   char error[CURL_ERROR_SIZE];
@@ -335,7 +342,17 @@ check_multi_info (OstreeFetcher *fetcher)
       else
         {
           curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);
-          if (!is_file && !(response >= 200 && response < 300))
+
+          if (!is_file && response == 304 &&
+              (req->if_none_match != NULL || req->if_modified_since > 0))
+            {
+              /* Version on the server is unchanged from the version we have
+               * cached locally; report this as an out-argument, a zero-length
+               * response buffer, and no error. */
+              req->out_not_modified = TRUE;
+            }
+
+          if (!is_file && !(response >= 200 && response < 300) && response != 304)
             {
               GIOErrorEnum giocode = _ostree_fetcher_http_status_code_to_io_error (response);
 
@@ -409,14 +426,11 @@ static gboolean
 timer_cb (gpointer data)
 {
   OstreeFetcher *fetcher = data;
-  GSource *orig_src = fetcher->timer_event;
-
+  g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source);
   (void)curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running);
   check_multi_info (fetcher);
-  if (fetcher->timer_event == orig_src)
-    fetcher->timer_event = NULL;
 
-  return FALSE;
+  return G_SOURCE_REMOVE;
 }
 
 /* Update the event timer after curl_multi library calls */
@@ -575,6 +589,40 @@ write_cb (void *ptr, size_t size, size_t nmemb, void *data)
   return realsize;
 }
 
+/* CURLOPT_HEADERFUNCTION */
+static size_t
+response_header_cb (const char *buffer, size_t size, size_t n_items, void *user_data)
+{
+  const size_t real_size = size * n_items;
+  GTask *task = G_TASK (user_data);
+  FetcherRequest *req;
+
+  /* libcurl says that @size is always 1, but let’s check
+   * See https://curl.haxx.se/libcurl/c/CURLOPT_HEADERFUNCTION.html */
+  g_assert (size == 1);
+
+  req = g_task_get_task_data (task);
+
+  const char *etag_header = "ETag: ";
+  const char *last_modified_header = "Last-Modified: ";
+
+  if (real_size > strlen (etag_header) &&
+      strncasecmp (buffer, etag_header, strlen (etag_header)) == 0)
+    {
+      g_clear_pointer (&req->out_etag, g_free);
+      req->out_etag = g_strstrip (g_strdup (buffer + strlen (etag_header)));
+    }
+  else if (real_size > strlen (last_modified_header) &&
+           strncasecmp (buffer, last_modified_header, strlen (last_modified_header)) == 0)
+    {
+      g_autofree char *lm_buf = g_strstrip (g_strdup (buffer + strlen (last_modified_header)));
+      g_autoptr(GDateTime) dt = _ostree_parse_rfc2616_date_time (lm_buf, strlen (lm_buf));
+      req->out_last_modified = (dt != NULL) ? g_date_time_to_unix (dt) : 0;
+    }
+
+  return real_size;
+}
+
 /* CURLOPT_PROGRESSFUNCTION */
 static int
 prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)
@@ -600,6 +648,9 @@ request_unref (FetcherRequest *req)
   glnx_tmpfile_clear (&req->tmpf);
   if (req->output_buf)
     g_string_free (req->output_buf, TRUE);
+  g_free (req->if_none_match);
+  g_free (req->out_etag);
+  g_clear_pointer (&req->req_headers, (GDestroyNotify)curl_slist_free_all);
   curl_easy_cleanup (req->easy);
 
   g_free (req);
@@ -721,8 +772,29 @@ initiate_next_curl_request (FetcherRequest *req,
 
   curl_easy_setopt (req->easy, CURLOPT_USERAGENT,
                     self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING);
-  if (self->extra_headers)
-    curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);
+
+  /* Set caching request headers */
+  if (req->if_none_match != NULL)
+    {
+      g_autofree char *if_none_match = g_strconcat ("If-None-Match: ", req->if_none_match, NULL);
+      req->req_headers = curl_slist_append (req->req_headers, if_none_match);
+    }
+
+  if (req->if_modified_since > 0)
+    {
+      g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (req->if_modified_since);
+      g_autofree char *mod_date = g_date_time_format (date_time, "If-Modified-Since: %a, %d %b %Y %H:%M:%S %Z");
+
+      req->req_headers = curl_slist_append (req->req_headers, mod_date);
+    }
+
+  /* Append a copy of @extra_headers to @req_headers, as the former could change
+   * between requests or while a request is in flight */
+  for (const struct curl_slist *l = self->extra_headers; l != NULL; l = l->next)
+    req->req_headers = curl_slist_append (req->req_headers, l->data);
+
+  if (req->req_headers != NULL)
+    curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, req->req_headers);
 
   if (self->cookie_jar_path)
     {
@@ -796,6 +868,7 @@ initiate_next_curl_request (FetcherRequest *req,
     }
 
   curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
+  curl_easy_setopt (req->easy, CURLOPT_HEADERFUNCTION, response_header_cb);
   if (g_getenv ("OSTREE_DEBUG_HTTP"))
     curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);
   curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error);
@@ -815,6 +888,7 @@ initiate_next_curl_request (FetcherRequest *req,
   /* closure bindings -> task */
   curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);
   curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);
+  curl_easy_setopt (req->easy, CURLOPT_HEADERDATA, task);
   curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);
 
   CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy);
@@ -826,6 +900,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
                                GPtrArray             *mirrorlist,
                                const char            *filename,
                                OstreeFetcherRequestFlags flags,
+                               const char            *if_none_match,
+                               guint64                if_modified_since,
                                gboolean               is_membuf,
                                guint64                max_size,
                                int                    priority,
@@ -859,6 +935,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
   req->filename = g_strdup (filename);
   req->max_size = max_size;
   req->flags = flags;
+  req->if_none_match = g_strdup (if_none_match);
+  req->if_modified_since = if_modified_since;
   req->is_membuf = is_membuf;
   /* We'll allocate the tmpfile on demand, so we handle
    * file I/O errors just in the write func.
@@ -882,13 +960,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
                                     GPtrArray             *mirrorlist,
                                     const char            *filename,
                                     OstreeFetcherRequestFlags flags,
+                                    const char            *if_none_match,
+                                    guint64                if_modified_since,
                                     guint64                max_size,
                                     int                    priority,
                                     GCancellable          *cancellable,
                                     GAsyncReadyCallback    callback,
                                     gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, FALSE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -897,6 +978,9 @@ gboolean
 _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                            GAsyncResult  *result,
                                            GLnxTmpfile   *out_tmpf,
+                                           gboolean      *out_not_modified,
+                                           char         **out_etag,
+                                           guint64       *out_last_modified,
                                            GError       **error)
 {
   g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
@@ -912,6 +996,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
   *out_tmpf = req->tmpf;
   req->tmpf.initialized = FALSE; /* Transfer ownership */
 
+  if (out_not_modified != NULL)
+    *out_not_modified = req->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&req->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = req->out_last_modified;
+
   return TRUE;
 }
 
@@ -920,13 +1011,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
                                    GPtrArray             *mirrorlist,
                                    const char            *filename,
                                    OstreeFetcherRequestFlags flags,
+                                   const char            *if_none_match,
+                                   guint64                if_modified_since,
                                    guint64                max_size,
                                    int                    priority,
                                    GCancellable          *cancellable,
                                    GAsyncReadyCallback    callback,
                                    gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, TRUE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -935,6 +1029,9 @@ gboolean
 _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                           GAsyncResult  *result,
                                           GBytes       **out_buf,
+                                          gboolean      *out_not_modified,
+                                          char         **out_etag,
+                                          guint64       *out_last_modified,
                                           GError       **error)
 {
   GTask *task;
@@ -955,6 +1052,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
   g_assert (out_buf);
   *out_buf = ret;
 
+  if (out_not_modified != NULL)
+    *out_not_modified = req->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&req->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = req->out_last_modified;
+
   return TRUE;
 }
 

src/libostree/ostree-fetcher-soup.c

@@ -49,7 +49,7 @@ typedef enum {
 } OstreeFetcherState;
 
 typedef struct {
-  volatile int ref_count;
+  int ref_count;  /* atomic */
 
   SoupSession *session;  /* not referenced */
   GMainContext *main_context;
@@ -77,7 +77,7 @@ typedef struct {
 } ThreadClosure;
 
 typedef struct {
-  volatile int ref_count;
+  int ref_count;  /* atomic */
 
   ThreadClosure *thread_closure;
   GPtrArray *mirrorlist; /* list of base URIs */
@@ -90,9 +90,14 @@ typedef struct {
 
   gboolean is_membuf;
   OstreeFetcherRequestFlags flags;
+  char *if_none_match;  /* request ETag */
+  guint64 if_modified_since;  /* seconds since the epoch */
   GInputStream *request_body;
   GLnxTmpfile tmpf;
   GOutputStream *out_stream;
+  gboolean out_not_modified;  /* TRUE if the server gave a HTTP 304 Not Modified response, which we don’t propagate as an error */
+  char *out_etag;  /* response ETag */
+  guint64 out_last_modified;  /* response Last-Modified, seconds since the epoch */
 
   guint64 max_size;
   guint64 current_size;
@@ -196,8 +201,10 @@ pending_uri_unref (OstreeFetcherPendingURI *pending)
   g_free (pending->filename);
   g_clear_object (&pending->request);
   g_clear_object (&pending->request_body);
+  g_free (pending->if_none_match);
   glnx_tmpfile_clear (&pending->tmpf);
   g_clear_object (&pending->out_stream);
+  g_free (pending->out_etag);
   g_free (pending);
 }
 
@@ -431,6 +438,22 @@ create_pending_soup_request (OstreeFetcherPendingURI  *pending,
 
   pending->request = soup_session_request_uri (pending->thread_closure->session,
                                                (SoupURI*)(uri ? uri : next_mirror), error);
+
+  /* Add caching headers. */
+  if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_none_match != NULL)
+    {
+      glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
+      soup_message_headers_append (msg->request_headers, "If-None-Match", pending->if_none_match);
+    }
+
+  if (SOUP_IS_REQUEST_HTTP (pending->request) && pending->if_modified_since > 0)
+    {
+      glnx_unref_object SoupMessage *msg = soup_request_http_get_message ((SoupRequestHTTP*) pending->request);
+
+      g_autoptr(GDateTime) date_time = g_date_time_new_from_unix_utc (pending->if_modified_since);
+      g_autofree char *mod_date = g_date_time_format (date_time, "%a, %d %b %Y %H:%M:%S %Z");
+      soup_message_headers_append (msg->request_headers, "If-Modified-Since", mod_date);
+    }
 }
 
 static void
@@ -1050,7 +1073,14 @@ on_request_sent (GObject        *object,
   if (SOUP_IS_REQUEST_HTTP (object))
     {
       msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
-      if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
+      if (msg->status_code == SOUP_STATUS_NOT_MODIFIED &&
+          (pending->if_none_match != NULL || pending->if_modified_since > 0))
+        {
+          /* Version on the server is unchanged from the version we have cached locally;
+           * report this as an out-argument, a zero-length response buffer, and no error */
+          pending->out_not_modified = TRUE;
+        }
+      else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
         {
           /* is there another mirror we can try? */
           if (pending->mirrorlist_idx + 1 < pending->mirrorlist->len)
@@ -1124,6 +1154,21 @@ on_request_sent (GObject        *object,
             }
           goto out;
         }
+
+      /* Grab cache properties from the response */
+      pending->out_etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag"));
+      pending->out_last_modified = 0;
+
+      const char *last_modified_str = soup_message_headers_get_one (msg->response_headers, "Last-Modified");
+      if (last_modified_str != NULL)
+        {
+          SoupDate *soup_date = soup_date_new_from_string (last_modified_str);
+          if (soup_date != NULL)
+            {
+              pending->out_last_modified = soup_date_to_time_t (soup_date);
+              soup_date_free (soup_date);
+            }
+        }
     }
 
   pending->state = OSTREE_FETCHER_STATE_DOWNLOADING;
@@ -1154,6 +1199,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
                                GPtrArray             *mirrorlist,
                                const char            *filename,
                                OstreeFetcherRequestFlags flags,
+                               const char            *if_none_match,
+                               guint64                if_modified_since,
                                gboolean               is_membuf,
                                guint64                max_size,
                                int                    priority,
@@ -1175,6 +1222,8 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
   pending->mirrorlist = g_ptr_array_ref (mirrorlist);
   pending->filename = g_strdup (filename);
   pending->flags = flags;
+  pending->if_none_match = g_strdup (if_none_match);
+  pending->if_modified_since = if_modified_since;
   pending->max_size = max_size;
   pending->is_membuf = is_membuf;
 
@@ -1196,13 +1245,16 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
                                     GPtrArray             *mirrorlist,
                                     const char            *filename,
                                     OstreeFetcherRequestFlags flags,
+                                    const char            *if_none_match,
+                                    guint64                if_modified_since,
                                     guint64                max_size,
                                     int                    priority,
                                     GCancellable          *cancellable,
                                     GAsyncReadyCallback    callback,
                                     gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, FALSE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, FALSE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -1211,6 +1263,9 @@ gboolean
 _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                            GAsyncResult  *result,
                                            GLnxTmpfile   *out_tmpf,
+                                           gboolean      *out_not_modified,
+                                           char         **out_etag,
+                                           guint64       *out_last_modified,
                                            GError       **error)
 {
   GTask *task;
@@ -1231,6 +1286,13 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
   *out_tmpf = pending->tmpf;
   pending->tmpf.initialized = FALSE; /* Transfer ownership */
 
+  if (out_not_modified != NULL)
+    *out_not_modified = pending->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&pending->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = pending->out_last_modified;
+
   return TRUE;
 }
 
@@ -1239,13 +1301,16 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
                                    GPtrArray             *mirrorlist,
                                    const char            *filename,
                                    OstreeFetcherRequestFlags flags,
+                                   const char            *if_none_match,
+                                   guint64                if_modified_since,
                                    guint64                max_size,
                                    int                    priority,
                                    GCancellable          *cancellable,
                                    GAsyncReadyCallback    callback,
                                    gpointer               user_data)
 {
-  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags,
+                                 if_none_match, if_modified_since, TRUE,
                                  max_size, priority, cancellable,
                                  callback, user_data);
 }
@@ -1254,6 +1319,9 @@ gboolean
 _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                           GAsyncResult  *result,
                                           GBytes       **out_buf,
+                                          gboolean      *out_not_modified,
+                                          char         **out_etag,
+                                          guint64       *out_last_modified,
                                           GError       **error)
 {
   GTask *task;
@@ -1274,6 +1342,13 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
   g_assert (out_buf);
   *out_buf = ret;
 
+  if (out_not_modified != NULL)
+    *out_not_modified = pending->out_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&pending->out_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = pending->out_last_modified;
+
   return TRUE;
 }
 

src/libostree/ostree-fetcher-util.c

@@ -34,6 +34,9 @@
 typedef struct
 {
   GBytes          *result_buf;
+  gboolean         result_not_modified;
+  char            *result_etag;
+  guint64          result_last_modified;  /* second since the epoch */
   gboolean         done;
   GError         **error;
 }
@@ -48,6 +51,8 @@ fetch_uri_sync_on_complete (GObject        *object,
 
   (void)_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)object,
                                                   result, &data->result_buf,
+                                                  &data->result_not_modified,
+                                                  &data->result_etag, &data->result_last_modified,
                                                   data->error);
   data->done = TRUE;
 }
@@ -57,7 +62,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once  (OstreeFetcher              *fe
                                                   GPtrArray                  *mirrorlist,
                                                   const char                 *filename,
                                                   OstreeFetcherRequestFlags   flags,
+                                                  const char                 *if_none_match,
+                                                  guint64                     if_modified_since,
                                                   GBytes                    **out_contents,
+                                                  gboolean                   *out_not_modified,
+                                                  char                      **out_etag,
+                                                  guint64                    *out_last_modified,
                                                   guint64                     max_size,
                                                   GCancellable               *cancellable,
                                                   GError                    **error)
@@ -79,7 +89,7 @@ _ostree_fetcher_mirrored_request_to_membuf_once  (OstreeFetcher              *fe
   data.error = error;
 
   _ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename,
-                                     flags,
+                                     flags, if_none_match, if_modified_since,
                                      max_size, OSTREE_FETCHER_DEFAULT_PRIORITY,
                                      cancellable, fetch_uri_sync_on_complete, &data);
   while (!data.done)
@@ -94,6 +104,12 @@ _ostree_fetcher_mirrored_request_to_membuf_once  (OstreeFetcher              *fe
               g_clear_error (error);
               ret = TRUE;
               *out_contents = NULL;
+              if (out_not_modified != NULL)
+                *out_not_modified = FALSE;
+              if (out_etag != NULL)
+                *out_etag = NULL;
+              if (out_last_modified != NULL)
+                *out_last_modified = 0;
             }
         }
       goto out;
@@ -101,10 +117,17 @@ _ostree_fetcher_mirrored_request_to_membuf_once  (OstreeFetcher              *fe
 
   ret = TRUE;
   *out_contents = g_steal_pointer (&data.result_buf);
+  if (out_not_modified != NULL)
+    *out_not_modified = data.result_not_modified;
+  if (out_etag != NULL)
+    *out_etag = g_steal_pointer (&data.result_etag);
+  if (out_last_modified != NULL)
+    *out_last_modified = data.result_last_modified;
  out:
   if (mainctx)
     g_main_context_pop_thread_default (mainctx);
   g_clear_pointer (&data.result_buf, (GDestroyNotify)g_bytes_unref);
+  g_clear_pointer (&data.result_etag, g_free);
   return ret;
 }
 
@@ -113,8 +136,13 @@ _ostree_fetcher_mirrored_request_to_membuf  (OstreeFetcher              *fetcher
                                              GPtrArray                  *mirrorlist,
                                              const char                 *filename,
                                              OstreeFetcherRequestFlags   flags,
+                                             const char                 *if_none_match,
+                                             guint64                     if_modified_since,
                                              guint                       n_network_retries,
                                              GBytes                    **out_contents,
+                                             gboolean                   *out_not_modified,
+                                             char                      **out_etag,
+                                             guint64                    *out_last_modified,
                                              guint64                     max_size,
                                              GCancellable               *cancellable,
                                              GError                    **error)
@@ -127,7 +155,9 @@ _ostree_fetcher_mirrored_request_to_membuf  (OstreeFetcher              *fetcher
       g_clear_error (&local_error);
       if (_ostree_fetcher_mirrored_request_to_membuf_once (fetcher, mirrorlist,
                                                            filename, flags,
-                                                           out_contents, max_size,
+                                                           if_none_match, if_modified_since,
+                                                           out_contents, out_not_modified, out_etag,
+                                                           out_last_modified, max_size,
                                                            cancellable, &local_error))
         return TRUE;
     }
@@ -143,8 +173,13 @@ gboolean
 _ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
                                        OstreeFetcherURI *uri,
                                        OstreeFetcherRequestFlags flags,
+                                       const char    *if_none_match,
+                                       guint64        if_modified_since,
                                        guint          n_network_retries,
                                        GBytes         **out_contents,
+                                       gboolean      *out_not_modified,
+                                       char         **out_etag,
+                                       guint64       *out_last_modified,
                                        guint64        max_size,
                                        GCancellable   *cancellable,
                                        GError         **error)
@@ -152,7 +187,9 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
   g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
   g_ptr_array_add (mirrorlist, uri); /* no transfer */
   return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL, flags,
-                                                     n_network_retries, out_contents, max_size,
+                                                     if_none_match, if_modified_since,
+                                                     n_network_retries, out_contents,
+                                                     out_not_modified, out_etag, out_last_modified, max_size,
                                                      cancellable, error);
 }
 

src/libostree/ostree-fetcher-util.h

@@ -56,8 +56,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
                                                      GPtrArray     *mirrorlist,
                                                      const char    *filename,
                                                      OstreeFetcherRequestFlags flags,
+                                                     const char    *if_none_match,
+                                                     guint64        if_modified_since,
                                                      guint          n_network_retries,
                                                      GBytes         **out_contents,
+                                                     gboolean      *out_not_modified,
+                                                     char         **out_etag,
+                                                     guint64       *out_last_modified,
                                                      guint64        max_size,
                                                      GCancellable   *cancellable,
                                                      GError         **error);
@@ -65,8 +70,13 @@ gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
 gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
                                                 OstreeFetcherURI *uri,
                                                 OstreeFetcherRequestFlags flags,
+                                                const char    *if_none_match,
+                                                guint64        if_modified_since,
                                                 guint          n_network_retries,
                                                 GBytes         **out_contents,
+                                                gboolean      *out_not_modified,
+                                                char         **out_etag,
+                                                guint64       *out_last_modified,
                                                 guint64        max_size,
                                                 GCancellable   *cancellable,
                                                 GError         **error);

src/libostree/ostree-fetcher.h

@@ -123,6 +123,8 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
                                          GPtrArray             *mirrorlist,
                                          const char            *filename,
                                          OstreeFetcherRequestFlags flags,
+                                         const char            *if_none_match,
+                                         guint64                if_modified_since,
                                          guint64                max_size,
                                          int                    priority,
                                          GCancellable          *cancellable,
@@ -132,12 +134,17 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
 gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                                     GAsyncResult  *result,
                                                     GLnxTmpfile   *out_tmpf,
+                                                    gboolean      *out_not_modified,
+                                                    char         **out_etag,
+                                                    guint64       *out_last_modified,
                                                     GError       **error);
 
 void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
                                         GPtrArray             *mirrorlist,
                                         const char            *filename,
                                         OstreeFetcherRequestFlags flags,
+                                        const char            *if_none_match,
+                                        guint64                if_modified_since,
                                         guint64                max_size,
                                         int                    priority,
                                         GCancellable          *cancellable,
@@ -147,6 +154,9 @@ void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
 gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                                    GAsyncResult  *result,
                                                    GBytes       **out_buf,
+                                                   gboolean      *out_not_modified,
+                                                   char         **out_etag,
+                                                   guint64       *out_last_modified,
                                                    GError       **error);
 
 

src/libostree/ostree-metalink.c

@@ -436,8 +436,10 @@ try_one_url (OstreeMetalinkRequest *self,
 
   if (!_ostree_fetcher_request_uri_to_membuf (self->metalink->fetcher,
                                               uri, 0,
+                                              NULL, 0,
                                               self->metalink->n_network_retries,
                                               &bytes,
+                                              NULL, NULL, NULL,
                                               self->metalink->max_size,
                                               self->cancellable,
                                               error))
@@ -618,8 +620,9 @@ _ostree_metalink_request_sync (OstreeMetalink        *self,
   request.parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, &request, NULL);
 
   if (!_ostree_fetcher_request_uri_to_membuf (self->fetcher, self->uri, 0,
+                                              NULL, 0,
                                               self->n_network_retries,
-                                              &contents, self->max_size,
+                                              &contents, NULL, NULL, NULL, self->max_size,
                                               cancellable, error))
     goto out;
 

src/libostree/ostree-remote-private.h

@@ -41,7 +41,7 @@ G_BEGIN_DECLS
  * remote which this one inherits from, and is what should be used in refspecs
  * for pulls from this remote. If it’s %NULL, @name should be used instead. */
 struct OstreeRemote {
-  volatile int ref_count;
+  int ref_count;  /* atomic */
   char *name;  /* (not nullable) */
   char *refspec_name;  /* (nullable) */
   char *group;   /* group name in options (not nullable) */

src/libostree/ostree-repo-private.h

@@ -57,6 +57,7 @@ G_BEGIN_DECLS
 #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map"
 #define OSTREE_SUMMARY_MODE "ostree.summary.mode"
 #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits"
+#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas"
 
 #define _OSTREE_PAYLOAD_LINK_PREFIX "../"
 #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1)
@@ -71,7 +72,7 @@ typedef enum {
 } OstreeRepoTestErrorFlags;
 
 struct OstreeRepoCommitModifier {
-  volatile gint refcount;
+  gint refcount;  /* atomic */
 
   OstreeRepoCommitModifierFlags flags;
   OstreeRepoCommitFilter filter;
@@ -109,6 +110,28 @@ typedef enum {
   _OSTREE_FEATURE_YES,
 } _OstreeFeatureSupport;
 
+/* Possible values for the sysroot.bootloader configuration variable */
+typedef enum {
+  CFG_SYSROOT_BOOTLOADER_OPT_AUTO = 0,
+  CFG_SYSROOT_BOOTLOADER_OPT_NONE,
+  CFG_SYSROOT_BOOTLOADER_OPT_GRUB2,
+  CFG_SYSROOT_BOOTLOADER_OPT_SYSLINUX,
+  CFG_SYSROOT_BOOTLOADER_OPT_UBOOT,
+  CFG_SYSROOT_BOOTLOADER_OPT_ZIPL,
+  /* Non-exhaustive */
+} OstreeCfgSysrootBootloaderOpt;
+
+static const char* const CFG_SYSROOT_BOOTLOADER_OPTS_STR[] = {
+  /* This must be kept in the same order as the enum */
+  "auto",
+  "none",
+  "grub2",
+  "syslinux",
+  "uboot",
+  "zipl",
+  NULL,
+};
+
 /**
  * OstreeRepo:
  *
@@ -192,7 +215,7 @@ struct OstreeRepo {
   guint64 payload_link_threshold;
   gint fs_support_reflink; /* The underlying filesystem has support for ioctl (FICLONE..) */
   gchar **repo_finders;
-  gchar *bootloader; /* Configure which bootloader to use. */
+  OstreeCfgSysrootBootloaderOpt bootloader; /* Configure which bootloader to use. */
 
   OstreeRepo *parent_repo;
 };

src/libostree/ostree-repo-pull-private.h

@@ -72,9 +72,15 @@ typedef struct {
   gboolean          has_tombstone_commits;
 
   GBytes           *summary_data;
+  char             *summary_etag;
+  guint64           summary_last_modified;  /* seconds since the epoch */
   GBytes           *summary_data_sig;
+  char             *summary_sig_etag;
+  guint64           summary_sig_last_modified;  /* seconds since the epoch */
   GVariant         *summary;
-  GHashTable       *summary_deltas_checksums;
+  GHashTable       *summary_deltas_checksums; /* Filled from summary and delta indexes */
+  gboolean          summary_has_deltas; /* True if the summary existed and had a delta index */
+  gboolean          has_indexed_deltas;
   GHashTable       *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
   GHashTable       *verified_commits; /* Set<checksum> of commits that have been verified */
   GHashTable       *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */
@@ -89,6 +95,7 @@ typedef struct {
   GHashTable       *requested_fallback_content; /* Maps checksum to itself */
   GHashTable       *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
   GHashTable       *pending_fetch_content; /* Map<checksum,FetchObjectData> */
+  GHashTable       *pending_fetch_delta_indexes; /* Set<FetchDeltaIndexData> */
   GHashTable       *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
   GHashTable       *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
   guint             n_outstanding_metadata_fetches;

src/libostree/ostree-repo-static-delta-compilation.c

@@ -1313,10 +1313,10 @@ get_fallback_headers (OstreeRepo               *self,
  * ostree_repo_static_delta_generate:
  * @self: Repo
  * @opt: High level optimization choice
- * @from: ASCII SHA256 checksum of origin, or %NULL
+ * @from: (nullable): ASCII SHA256 checksum of origin, or %NULL
  * @to: ASCII SHA256 checksum of target
- * @metadata: (allow-none): Optional metadata
- * @params: (allow-none): Parameters, see below
+ * @metadata: (nullable): Optional metadata
+ * @params: (nullable): Parameters, see below
  * @cancellable: Cancellable
  * @error: Error
  *

src/libostree/ostree-repo-static-delta-core.c

@@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo                  *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_list_static_delta_indexes:
+ * @self: Repo
+ * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * This function synchronously enumerates all static delta indexes in the
+ * repository, returning its result in @out_indexes.
+ *
+ * Since: 2020.7
+ */
+gboolean
+ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                       GPtrArray                  **out_indexes,
+                                       GCancellable                *cancellable,
+                                       GError                     **error)
+{
+  g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free);
+
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  gboolean exists;
+  if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter,
+                                     &exists, error))
+    return FALSE;
+  if (!exists)
+    {
+      /* Note early return */
+      ot_transfer_out_value (out_indexes, &ret_indexes);
+      return TRUE;
+    }
+
+  while (TRUE)
+    {
+      g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, };
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
+        return FALSE;
+      if (dent == NULL)
+        break;
+      if (dent->d_type != DT_DIR)
+        continue;
+      if (strlen (dent->d_name) != 2)
+        continue;
+
+      if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE,
+                                        &sub_dfd_iter, error))
+        return FALSE;
+
+      while (TRUE)
+        {
+          struct dirent *sub_dent;
+
+          if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent,
+                                                           cancellable, error))
+            return FALSE;
+          if (sub_dent == NULL)
+            break;
+          if (sub_dent->d_type != DT_REG)
+            continue;
+
+          const char *name1 = dent->d_name;
+          const char *name2 = sub_dent->d_name;
+
+          /* base64 len is 43, but 2 chars are in the parent dir name */
+          if (strlen (name2) != 41 + strlen(".index") ||
+              !g_str_has_suffix (name2, ".index"))
+            continue;
+
+          g_autoptr(GString) out = g_string_new (name1);
+          g_string_append_len (out, name2, 41);
+
+          char checksum[OSTREE_SHA256_STRING_LEN+1];
+          guchar csum[OSTREE_SHA256_DIGEST_LEN];
+
+          ostree_checksum_b64_inplace_to_bytes (out->str, csum);
+          ostree_checksum_inplace_from_bytes (csum, checksum);
+
+          g_ptr_array_add (ret_indexes, g_strdup (checksum));
+        }
+    }
+
+  ot_transfer_out_value (out_indexes, &ret_indexes);
+  return TRUE;
+}
+
 gboolean
 _ostree_repo_static_delta_part_have_all_objects (OstreeRepo             *repo,
                                                  GVariant               *checksum_array,
@@ -508,7 +595,7 @@ ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo   *self,
            */
           delta_open_flags |= OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM;
 
-          if (!_ostree_static_delta_part_open (part_in, inline_part_bytes, 
+          if (!_ostree_static_delta_part_open (part_in, inline_part_bytes,
                                                delta_open_flags,
                                                NULL,
                                                &part,
@@ -1076,7 +1163,7 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
  * @self: Repo
  * @delta_id: delta path
  * @sign: Signature engine used to check superblock
- * @out_success_message: success message
+ * @out_success_message: (out) (nullable) (optional): success message
  * @error: Error
  *
  * Verify static delta file signature.
@@ -1118,3 +1205,207 @@ ostree_repo_static_delta_verify_signature (OstreeRepo       *self,
 
   return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error);
 }
+
+static void
+null_or_ptr_array_unref (GPtrArray *array)
+{
+  if (array != NULL)
+    g_ptr_array_unref (array);
+}
+
+static gboolean
+file_has_content (OstreeRepo   *repo,
+                  const char   *subpath,
+                  GBytes       *data,
+                  GCancellable *cancellable)
+{
+  struct stat stbuf;
+  glnx_autofd int existing_fd = -1;
+
+  if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL))
+    return FALSE;
+
+  if (stbuf.st_size != g_bytes_get_size (data))
+    return FALSE;
+
+  if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL))
+    return FALSE;
+
+  g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL);
+  if (existing_data == NULL)
+    return FALSE;
+
+  return g_bytes_equal (existing_data, data);
+}
+
+/**
+ * ostree_repo_static_delta_reindex:
+ * @repo: Repo
+ * @flags: Flags affecting the indexing operation
+ * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * The delta index for a particular commit lists all the existing deltas that can be used
+ * when downloading that commit. This operation regenerates these indexes, either for
+ * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that
+ * are reachable by an existing delta (if @opt_to_commit is %NULL).
+ *
+ * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary().
+ *
+ * Locking: shared
+ */
+gboolean
+ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                  OstreeStaticDeltaIndexFlags flags,
+                                  const char                 *opt_to_commit,
+                                  GCancellable               *cancellable,
+                                  GError                    **error)
+{
+  g_autoptr(GPtrArray) all_deltas = NULL;
+  g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */
+  gboolean opt_indexed_deltas;
+
+  /* Protect against parallel prune operation */
+  g_autoptr(OstreeRepoAutoLock) lock =
+    _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error);
+  if (!lock)
+    return FALSE;
+
+  /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */
+  if (!ot_keyfile_get_boolean_with_default (repo->config, "core",
+                                            "indexed-deltas", FALSE,
+                                            &opt_indexed_deltas, error))
+    return FALSE;
+
+  if (!opt_indexed_deltas)
+    {
+      g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo);
+      g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE);
+      if (!ostree_repo_write_config (repo, config, error))
+        return FALSE;
+    }
+
+  deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref);
+
+  if (opt_to_commit == NULL)
+    {
+      g_autoptr(GPtrArray) old_indexes = NULL;
+
+      /* To ensure all old index files either is regenerated, or
+       * removed, we initialize all existing indexes to NULL in the
+       * hashtable. */
+      if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error))
+        return FALSE;
+
+      for (int i = 0; i < old_indexes->len; i++)
+        {
+          const char *old_index = g_ptr_array_index (old_indexes, i);
+          g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL);
+        }
+    }
+  else
+    {
+      if (!ostree_validate_checksum_string (opt_to_commit, error))
+        return FALSE;
+
+      /* We ensure the specific old index either is regenerated, or removed */
+      g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL);
+    }
+
+  if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error))
+    return FALSE;
+
+  for (int i = 0; i < all_deltas->len; i++)
+    {
+      const char *delta_name = g_ptr_array_index (all_deltas, i);
+      g_autofree char *from = NULL;
+      g_autofree char *to = NULL;
+      GPtrArray *deltas_to_commit = NULL;
+
+      if (!_ostree_parse_delta_name (delta_name, &from, &to, error))
+        return FALSE;
+
+      if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0)
+        continue;
+
+      deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to);
+      if (deltas_to_commit == NULL)
+        {
+          deltas_to_commit = g_ptr_array_new_with_free_func (g_free);
+          g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit);
+        }
+
+      g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from));
+    }
+
+  GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms)
+    {
+      g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to);
+
+      if (froms == NULL)
+        {
+          /* No index to this checksum seen, delete if it exists */
+
+          g_debug ("Removing delta index for %s", to);
+          if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error))
+            return FALSE;
+        }
+      else
+        {
+          g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+          g_autoptr(GVariant) index_variant = NULL;
+          g_autoptr(GBytes) index = NULL;
+
+          /* We sort on from here so that the index file is reproducible */
+          g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0);
+
+          g_variant_dict_init (&deltas_builder, NULL);
+
+          for (int i = 0; i < froms->len; i++)
+            {
+              const char *from = g_ptr_array_index (froms, i);
+              g_autofree char *delta_name = NULL;
+              GVariant *digest;
+
+              digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error);
+              if (digest == NULL)
+                return FALSE;
+
+              if (from != NULL)
+                delta_name = g_strconcat (from, "-", to, NULL);
+              else
+                delta_name = g_strdup (to);
+
+              g_variant_dict_insert_value (&deltas_builder, delta_name, digest);
+            }
+
+          /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when
+           * storing deltas in the summary. */
+          g_variant_dict_init (&index_builder, NULL);
+
+          g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+
+          index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder));
+          index = g_variant_get_data_as_bytes (index_variant);
+
+          g_autofree char *index_dirname = g_path_get_dirname (index_path);
+          if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error))
+            return FALSE;
+
+          /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will
+             lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */
+          if (file_has_content (repo, index_path, index, cancellable))
+            continue;
+
+          g_debug ("Updating delta index for %s", to);
+          if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path,
+                                              g_bytes_get_data (index, NULL), g_bytes_get_size (index),
+                                              0, cancellable, error))
+            return FALSE;
+        }
+    }
+
+  return TRUE;
+}

src/libostree/ostree-repo-static-delta-private.h

@@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo                 *repo,
                                   const char                 *delta_id,
                                   GCancellable               *cancellable,
                                   GError                    **error);
+gboolean
+_ostree_repo_static_delta_reindex (OstreeRepo                 *repo,
+                                   const char                 *opt_to_commit,
+                                   GCancellable               *cancellable,
+                                   GError                    **error);
 
 /* Used for static deltas which due to a historical mistake are
  * inconsistent endian.

src/libostree/ostree-repo.c

@@ -1048,7 +1048,6 @@ ostree_repo_finalize (GObject *object)
   g_mutex_clear (&self->txn_lock);
   g_free (self->collection_id);
   g_strfreev (self->repo_finders);
-  g_free (self->bootloader);
 
   g_clear_pointer (&self->remotes, g_hash_table_destroy);
   g_mutex_clear (&self->remotes_lock);
@@ -3186,28 +3185,28 @@ reload_sysroot_config (OstreeRepo          *self,
                        GCancellable        *cancellable,
                        GError             **error)
 {
-  { g_autofree char *bootloader = NULL;
+  g_autofree char *bootloader = NULL;
 
-    if (!ot_keyfile_get_value_with_default_group_optional (self->config, "sysroot",
-                                                           "bootloader", "auto",
-                                                           &bootloader, error))
-      return FALSE;
+  if (!ot_keyfile_get_value_with_default_group_optional (self->config, "sysroot",
+                                                         "bootloader", "auto",
+                                                         &bootloader, error))
+    return FALSE;
 
-    /* TODO: possibly later add support for specifying a generic bootloader
-     * binary "x" in /usr/lib/ostree/bootloaders/x). See:
-     * https://github.com/ostreedev/ostree/issues/1719
-     * https://github.com/ostreedev/ostree/issues/1801
-     * Also, dedup these strings with the bootloader implementations
-     */
-    if (!(g_str_equal (bootloader, "auto") || g_str_equal (bootloader, "none")
-          || g_str_equal (bootloader, "zipl")))
-      return glnx_throw (error, "Invalid bootloader configuration: '%s'", bootloader);
+  /* TODO: possibly later add support for specifying a generic bootloader
+   * binary "x" in /usr/lib/ostree/bootloaders/x). See:
+   * https://github.com/ostreedev/ostree/issues/1719
+   * https://github.com/ostreedev/ostree/issues/1801
+   */
+  for (int i = 0; CFG_SYSROOT_BOOTLOADER_OPTS_STR[i]; i++)
+    {
+      if (g_str_equal (bootloader, CFG_SYSROOT_BOOTLOADER_OPTS_STR[i]))
+        {
+          self->bootloader = (OstreeCfgSysrootBootloaderOpt) i;
+          return TRUE;
+        }
+    }
 
-    g_free (self->bootloader);
-    self->bootloader = g_steal_pointer (&bootloader);
-  }
-
-  return TRUE;
+  return glnx_throw (error, "Invalid bootloader configuration: '%s'", bootloader);
 }
 
 /**
@@ -5222,6 +5221,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo     *self,
 #endif /* OSTREE_DISABLE_GPGME */
 }
 
+
+/**
+ * ostree_repo_gpg_sign_data:
+ * @self: Self
+ * @data: Data as a #GBytes
+ * @old_signatures: Existing signatures to append to (or %NULL)
+ * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys.
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @out_signatures: (out): in case of success will contain signature
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Sign the given @data with the specified keys in @key_id. Similar to
+ * ostree_repo_add_gpg_signature_summary() but can be used on any
+ * data.
+ *
+ * You can use ostree_repo_gpg_verify_data() to verify the signatures.
+ *
+ * Returns: @TRUE if @data has been signed successfully,
+ * @FALSE in case of error (@error will contain the reason).
+ *
+ * Since: 2020.8
+ */
+gboolean
+ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                           GBytes         *data,
+                           GBytes         *old_signatures,
+                           const gchar   **key_id,
+                           const gchar    *homedir,
+                           GBytes        **out_signatures,
+                           GCancellable   *cancellable,
+                           GError        **error)
+{
+#ifndef OSTREE_DISABLE_GPGME
+  g_autoptr(GVariant) metadata = NULL;
+  g_autoptr(GVariant) res = NULL;
+
+  if (old_signatures)
+    metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE));
+
+  for (guint i = 0; key_id[i]; i++)
+    {
+      g_autoptr(GBytes) signature_data = NULL;
+      if (!sign_data (self, data, key_id[i], homedir,
+                      &signature_data,
+                      cancellable, error))
+        return FALSE;
+
+      g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
+      metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data);
+    }
+
+  res = g_variant_get_normal_form (metadata);
+  *out_signatures = g_variant_get_data_as_bytes (res);
+  return TRUE;
+#else
+  return glnx_throw (error, "GPG feature is disabled in a build time");
+#endif /* OSTREE_DISABLE_GPGME */
+}
+
+
 #ifndef OSTREE_DISABLE_GPGME
 /* Special remote for _ostree_repo_gpg_verify_with_metadata() */
 static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__";
@@ -5749,6 +5809,8 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
    * commits from working.
    */
   g_autoptr(OstreeRepoAutoLock) lock = NULL;
+  gboolean no_deltas_in_summary = FALSE;
+
   lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE,
                                       cancellable, error);
   if (!lock)
@@ -5781,35 +5843,41 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
       }
   }
 
-  {
-    g_autoptr(GPtrArray) delta_names = NULL;
-    g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
+  if (!ot_keyfile_get_boolean_with_default (self->config, "core",
+                                            "no-deltas-in-summary", FALSE,
+                                            &no_deltas_in_summary, error))
+    return FALSE;
 
-    if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
-      return FALSE;
+  if (!no_deltas_in_summary)
+    {
+      g_autoptr(GPtrArray) delta_names = NULL;
+      g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER;
 
-    g_variant_dict_init (&deltas_builder, NULL);
-    for (guint i = 0; i < delta_names->len; i++)
-      {
-        g_autofree char *from = NULL;
-        g_autofree char *to = NULL;
-        GVariant *digest;
+      if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error))
+        return FALSE;
 
-        if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
-          return FALSE;
+      g_variant_dict_init (&deltas_builder, NULL);
+      for (guint i = 0; i < delta_names->len; i++)
+        {
+          g_autofree char *from = NULL;
+          g_autofree char *to = NULL;
+          GVariant *digest;
 
-        digest = _ostree_repo_static_delta_superblock_digest (self,
-                                                              (from && from[0]) ? from : NULL,
-                                                              to, cancellable, error);
-        if (digest == NULL)
-          return FALSE;
+          if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
+            return FALSE;
 
-        g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
-      }
+          digest = _ostree_repo_static_delta_superblock_digest (self,
+                                                                (from && from[0]) ? from : NULL,
+                                                                to, cancellable, error);
+          if (digest == NULL)
+            return FALSE;
 
-    if (delta_names->len > 0)
-      g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
-  }
+          g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest);
+        }
+
+      if (delta_names->len > 0)
+        g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder));
+    }
 
   {
     g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED,
@@ -5834,6 +5902,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
                                  g_variant_new_boolean (tombstone_commits));
   }
 
+  g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS,
+                               g_variant_new_boolean (TRUE));
+
   /* Add refs which have a collection specified, which could be in refs/mirrors,
    * refs/heads, and/or refs/remotes. */
   {
@@ -5927,6 +5998,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
     g_variant_ref_sink (summary);
   }
 
+  if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error))
+    return FALSE;
+
   if (!_ostree_repo_file_replace_contents (self,
                                            self->repo_dir_fd,
                                            "summary",
@@ -6248,7 +6322,7 @@ ostree_repo_get_default_repo_finders (OstreeRepo *self)
  * Get the bootloader configured. See the documentation for the
  * "sysroot.bootloader" config key.
  *
- * Returns: bootloader configuration for the sysroot
+ * Returns: (transfer none): bootloader configuration for the sysroot
  * Since: 2019.2
  */
 const gchar *
@@ -6256,7 +6330,7 @@ ostree_repo_get_bootloader (OstreeRepo   *self)
 {
   g_return_val_if_fail (OSTREE_IS_REPO (self), NULL);
 
-  return self->bootloader;
+  return CFG_SYSROOT_BOOTLOADER_OPTS_STR[self->bootloader];
 }
 
 

src/libostree/ostree-repo.h

@@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo                  *self,
                                               GCancellable                *cancellable,
                                               GError                     **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_list_static_delta_indexes (OstreeRepo                  *self,
+                                                GPtrArray                  **out_indexes,
+                                                GCancellable                *cancellable,
+                                                GError                     **error);
+
 /**
  * OstreeStaticDeltaGenerateOpt:
  * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space
@@ -1068,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                             GCancellable                 *cancellable,
                                             GError                      **error);
 
+/**
+ * OstreeStaticDeltaIndexFlags:
+ * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags
+ *
+ * Flags controlling static delta index generation.
+ */
+typedef enum {
+  OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0,
+} OstreeStaticDeltaIndexFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_static_delta_reindex (OstreeRepo                  *repo,
+                                           OstreeStaticDeltaIndexFlags flags,
+                                           const char                  *opt_to_commit,
+                                           GCancellable                *cancellable,
+                                           GError                     **error);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo   *self,
                                                                   GFile        *dir_or_file,
@@ -1393,6 +1416,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo     *self,
                                            GCancellable   *cancellable,
                                            GError        **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_repo_gpg_sign_data (OstreeRepo     *self,
+                                    GBytes         *data,
+                                    GBytes         *old_signatures,
+                                    const gchar   **key_id,
+                                    const gchar    *homedir,
+                                    GBytes        **out_signatures,
+                                    GCancellable   *cancellable,
+                                    GError        **error);
+
 _OSTREE_PUBLIC
 OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo    *self,
                                                        const gchar   *commit_checksum,

src/libostree/ostree-sysroot-deploy.c

@@ -44,7 +44,6 @@
 #include "ostree-repo-private.h"
 #include "ostree-sysroot-private.h"
 #include "ostree-sepolicy-private.h"
-#include "ostree-bootloader-zipl.h"
 #include "ostree-deployment-private.h"
 #include "ostree-core-private.h"
 #include "ostree-linuxfsutil.h"
@@ -2561,7 +2560,6 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
   gboolean bootloader_is_atomic = FALSE;
   SyncStats syncstats = { 0, };
   g_autoptr(OstreeBootloader) bootloader = NULL;
-  const char *bootloader_config = NULL;
   if (!requires_new_bootversion)
     {
       if (!create_new_bootlinks (self, self->bootversion,
@@ -2593,29 +2591,8 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
             return glnx_throw_errno_prefix (error, "Remounting /boot read-write");
         }
 
-      OstreeRepo *repo = ostree_sysroot_repo (self);
-
-      bootloader_config = ostree_repo_get_bootloader (repo);
-
-      g_debug ("Using bootloader configuration: %s", bootloader_config);
-
-      if (g_str_equal (bootloader_config, "auto"))
-        {
-          if (!_ostree_sysroot_query_bootloader (self, &bootloader, cancellable, error))
-            return FALSE;
-        }
-      else if (g_str_equal (bootloader_config, "none"))
-        {
-          /* No bootloader specified; do not query bootloaders to run. */
-        }
-      else if (g_str_equal (bootloader_config, "zipl"))
-        {
-          /* Because we do not mark zipl as active by default, lets creating one here,
-           * which is basically the same what _ostree_sysroot_query_bootloader() does
-           * for other bootloaders if being activated.
-           * */
-          bootloader = (OstreeBootloader*) _ostree_bootloader_zipl_new (self);
-        }
+      if (!_ostree_sysroot_query_bootloader (self, &bootloader, cancellable, error))
+        return FALSE;
 
       bootloader_is_atomic = bootloader != NULL && _ostree_bootloader_is_atomic (bootloader);
 
@@ -2646,6 +2623,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
                        (bootloader_is_atomic ? "Transaction complete" : "Bootloader updated"),
                        requires_new_bootversion ? "yes" : "no",
                        new_deployments->len - self->deployments->len);
+    const gchar *bootloader_config = ostree_repo_get_bootloader (ostree_sysroot_repo (self));
     ot_journal_send ("MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_DEPLOYMENT_COMPLETE_ID),
                      "MESSAGE=%s", msg,
                      "OSTREE_BOOTLOADER=%s", bootloader ? _ostree_bootloader_get_name (bootloader) : "none",
@@ -3046,7 +3024,7 @@ _ostree_sysroot_deserialize_deployment_from_variant (GVariant *v,
 /**
  * ostree_sysroot_stage_overlay_initrd:
  * @self: Sysroot
- * @fd: (transfer none): File descriptor to overlay initrd
+ * @fd: File descriptor to overlay initrd
  * @out_checksum: (out) (transfer full): Overlay initrd checksum
  * @cancellable: Cancellable
  * @error: Error

src/libostree/ostree-sysroot-upgrader.c

@@ -682,9 +682,9 @@ ostree_sysroot_upgrader_deploy (OstreeSysrootUpgrader  *self,
 GType
 ostree_sysroot_upgrader_flags_get_type (void)
 {
-  static volatile gsize g_define_type_id__volatile = 0;
+  static gsize static_g_define_type_id = 0;
 
-  if (g_once_init_enter (&g_define_type_id__volatile))
+  if (g_once_init_enter (&static_g_define_type_id))
     {
       static const GFlagsValue values[] = {
         { OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED, "OSTREE_SYSROOT_UPGRADER_FLAGS_IGNORE_UNCONFIGURED", "ignore-unconfigured" },
@@ -692,8 +692,8 @@ ostree_sysroot_upgrader_flags_get_type (void)
       };
       GType g_define_type_id =
         g_flags_register_static (g_intern_static_string ("OstreeSysrootUpgraderFlags"), values);
-      g_once_init_leave (&g_define_type_id__volatile, g_define_type_id);
+      g_once_init_leave (&static_g_define_type_id, g_define_type_id);
     }
 
-  return g_define_type_id__volatile;
+  return static_g_define_type_id;
 }

src/libostree/ostree-sysroot.c

@@ -36,6 +36,7 @@
 #include "ostree-bootloader-uboot.h"
 #include "ostree-bootloader-syslinux.h"
 #include "ostree-bootloader-grub2.h"
+#include "ostree-bootloader-zipl.h"
 
 /**
  * SECTION:ostree-sysroot
@@ -256,9 +257,9 @@ ostree_sysroot_set_mount_namespace_in_use (OstreeSysroot  *self)
 
 /**
  * ostree_sysroot_get_path:
- * @self:
+ * @self: Sysroot
  *
- * Returns: (transfer none): Path to rootfs
+ * Returns: (transfer none) (not nullable): Path to rootfs
  */
 GFile *
 ostree_sysroot_get_path (OstreeSysroot  *self)
@@ -1185,7 +1186,7 @@ ostree_sysroot_get_subbootversion (OstreeSysroot   *self)
  * ostree_sysroot_get_booted_deployment:
  * @self: Sysroot
  *
- * Returns: (transfer none): The currently booted deployment, or %NULL if none
+ * Returns: (transfer none) (nullable): The currently booted deployment, or %NULL if none
  */
 OstreeDeployment *
 ostree_sysroot_get_booted_deployment (OstreeSysroot       *self)
@@ -1199,7 +1200,7 @@ ostree_sysroot_get_booted_deployment (OstreeSysroot       *self)
  * ostree_sysroot_get_staged_deployment:
  * @self: Sysroot
  *
- * Returns: (transfer none): The currently staged deployment, or %NULL if none
+ * Returns: (transfer none) (nullable): The currently staged deployment, or %NULL if none
  *
  * Since: 2018.5
  */
@@ -1237,7 +1238,7 @@ ostree_sysroot_get_deployments (OstreeSysroot  *self)
  * to access, it, you must either use fd-relative api such as openat(),
  * or concatenate it with the full ostree_sysroot_get_path().
  *
- * Returns: (transfer full): Path to deployment root directory, relative to sysroot
+ * Returns: (transfer full) (not nullable): Path to deployment root directory, relative to sysroot
  */
 char *
 ostree_sysroot_get_deployment_dirpath (OstreeSysroot    *self,
@@ -1312,7 +1313,7 @@ ostree_sysroot_get_repo (OstreeSysroot         *self,
  * returns a cached repository. Can only be called after ostree_sysroot_initialize()
  * or ostree_sysroot_load() has been invoked successfully.
  *
- * Returns: (transfer none): The OSTree repository in sysroot @self.
+ * Returns: (transfer none) (not nullable): The OSTree repository in sysroot @self.
  *
  * Since: 2017.7
  */
@@ -1324,10 +1325,38 @@ ostree_sysroot_repo (OstreeSysroot *self)
   return self->repo;
 }
 
+static OstreeBootloader*
+_ostree_sysroot_new_bootloader_by_type (
+    OstreeSysroot                 *sysroot,
+    OstreeCfgSysrootBootloaderOpt  bl_type)
+{
+  switch (bl_type)
+    {
+    case CFG_SYSROOT_BOOTLOADER_OPT_NONE:
+      /* No bootloader specified; do not query bootloaders to run. */
+      return NULL;
+    case CFG_SYSROOT_BOOTLOADER_OPT_GRUB2:
+      return (OstreeBootloader*) _ostree_bootloader_grub2_new (sysroot);
+    case CFG_SYSROOT_BOOTLOADER_OPT_SYSLINUX:
+      return (OstreeBootloader*) _ostree_bootloader_syslinux_new (sysroot);
+    case CFG_SYSROOT_BOOTLOADER_OPT_UBOOT:
+      return (OstreeBootloader*) _ostree_bootloader_uboot_new (sysroot);
+    case CFG_SYSROOT_BOOTLOADER_OPT_ZIPL:
+      /* We never consider zipl as active by default, so it can only be created
+       * if it's explicitly requested in the config */
+      return (OstreeBootloader*) _ostree_bootloader_zipl_new (sysroot);
+    case CFG_SYSROOT_BOOTLOADER_OPT_AUTO:
+      /* "auto" is handled by ostree_sysroot_query_bootloader so we should
+       * never get here: Fallthrough */
+    default:
+      g_assert_not_reached ();
+    }
+}
+
 /**
  * ostree_sysroot_query_bootloader:
  * @sysroot: Sysroot
- * @out_bootloader: (out) (transfer full) (allow-none): Return location for bootloader, may be %NULL
+ * @out_bootloader: (out) (transfer full) (optional) (nullable): Return location for bootloader, may be %NULL
  * @cancellable: Cancellable
  * @error: Error
  */
@@ -1337,32 +1366,38 @@ _ostree_sysroot_query_bootloader (OstreeSysroot     *sysroot,
                                   GCancellable      *cancellable,
                                   GError           **error)
 {
-  gboolean is_active;
-  g_autoptr(OstreeBootloader) ret_loader =
-    (OstreeBootloader*)_ostree_bootloader_syslinux_new (sysroot);
-  if (!_ostree_bootloader_query (ret_loader, &is_active,
-                                 cancellable, error))
-    return FALSE;
+  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
+  OstreeCfgSysrootBootloaderOpt bootloader_config = repo->bootloader;
 
-  if (!is_active)
-    {
-      g_object_unref (ret_loader);
-      ret_loader = (OstreeBootloader*)_ostree_bootloader_grub2_new (sysroot);
-      if (!_ostree_bootloader_query (ret_loader, &is_active,
-                                     cancellable, error))
-        return FALSE;
-    }
-  if (!is_active)
-    {
-      g_object_unref (ret_loader);
-      ret_loader = (OstreeBootloader*)_ostree_bootloader_uboot_new (sysroot);
-      if (!_ostree_bootloader_query (ret_loader, &is_active, cancellable, error))
-        return FALSE;
-    }
-  if (!is_active)
-    g_clear_object (&ret_loader);
+  g_debug ("Using bootloader configuration: %s",
+      CFG_SYSROOT_BOOTLOADER_OPTS_STR[bootloader_config]);
 
-  ot_transfer_out_value(out_bootloader, &ret_loader);
+  g_autoptr(OstreeBootloader) ret_loader = NULL;
+  if (bootloader_config == CFG_SYSROOT_BOOTLOADER_OPT_AUTO)
+    {
+      OstreeCfgSysrootBootloaderOpt probe[] = {
+        CFG_SYSROOT_BOOTLOADER_OPT_SYSLINUX,
+        CFG_SYSROOT_BOOTLOADER_OPT_GRUB2,
+        CFG_SYSROOT_BOOTLOADER_OPT_UBOOT,
+      };
+      for (int i = 0; i < G_N_ELEMENTS (probe); i++)
+        {
+          g_autoptr(OstreeBootloader) bl = _ostree_sysroot_new_bootloader_by_type (
+              sysroot, probe[i]);
+          gboolean is_active = FALSE;
+          if (!_ostree_bootloader_query (bl, &is_active, cancellable, error))
+            return FALSE;
+          if (is_active)
+            {
+              ret_loader = g_steal_pointer (&bl);
+              break;
+            }
+        }
+    }
+  else
+    ret_loader = _ostree_sysroot_new_bootloader_by_type (sysroot, bootloader_config);
+
+  ot_transfer_out_value (out_bootloader, &ret_loader)
   return TRUE;
 }
 
@@ -1393,8 +1428,8 @@ _ostree_sysroot_join_lines (GPtrArray  *lines)
  * ostree_sysroot_query_deployments_for:
  * @self: Sysroot
  * @osname: (allow-none): "stateroot" name
- * @out_pending: (out) (allow-none) (transfer full): The pending deployment
- * @out_rollback: (out) (allow-none) (transfer full): The rollback deployment
+ * @out_pending: (out) (nullable) (optional) (transfer full): The pending deployment
+ * @out_rollback: (out) (nullable) (optional) (transfer full): The rollback deployment
  *
  * Find the pending and rollback deployments for @osname. Pass %NULL for @osname
  * to use the booted deployment's osname. By default, pending deployment is the
@@ -1454,7 +1489,7 @@ ostree_sysroot_query_deployments_for (OstreeSysroot     *self,
  * Find the deployment to use as a configuration merge source; this is
  * the first one in the current deployment list which matches osname.
  *
- * Returns: (transfer full): Configuration merge deployment
+ * Returns: (transfer full) (nullable): Configuration merge deployment
  */
 OstreeDeployment *
 ostree_sysroot_get_merge_deployment (OstreeSysroot     *self,
@@ -1485,7 +1520,7 @@ ostree_sysroot_get_merge_deployment (OstreeSysroot     *self,
  * @self: Sysroot
  * @refspec: A refspec
  *
- * Returns: (transfer full): A new config file which sets @refspec as an origin
+ * Returns: (transfer full) (not nullable): A new config file which sets @refspec as an origin
  */
 GKeyFile *
 ostree_sysroot_origin_new_from_refspec (OstreeSysroot  *self,

src/libostree/ostree-version.h

@@ -43,7 +43,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_RELEASE_VERSION (7)
+#define OSTREE_RELEASE_VERSION (8)
 
 /**
  * OSTREE_VERSION
@@ -52,7 +52,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION (2020.7)
+#define OSTREE_VERSION (2020.8)
 
 /**
  * OSTREE_VERSION_S:
@@ -62,7 +62,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION_S "2020.7"
+#define OSTREE_VERSION_S "2020.8"
 
 #define OSTREE_ENCODE_VERSION(year,release) \
         ((year) << 16 | (release))
@@ -100,4 +100,4 @@
  *
  * Since: 2019.3
  */
-#define OSTREE_BUILT_FEATURES "libcurl libsoup gpgme ex-fsverity libarchive selinux openssl libmount systemd release p2p"
+#define OSTREE_BUILT_FEATURES "libsoup gpgme ex-fsverity libarchive selinux avahi libmount systemd release p2p"

src/ostree/ostree-trivial-httpd.c

@@ -206,6 +206,15 @@ close_socket (SoupMessage *msg, gpointer user_data)
 #endif
 }
 
+/* Returns the ETag including the surrounding quotes */
+static gchar *
+calculate_etag (GMappedFile *mapping)
+{
+  g_autoptr(GBytes) bytes = g_mapped_file_get_bytes (mapping);
+  g_autofree gchar *checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, bytes);
+  return g_strconcat ("\"", checksum, "\"", NULL);
+}
+
 static void
 do_get (OtTrivialHttpd    *self,
         SoupServer        *server,
@@ -367,31 +376,41 @@ do_get (OtTrivialHttpd    *self,
           soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
           goto out;
         }
+
+      glnx_autofd int fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
+      if (fd < 0)
+        {
+          soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+          goto out;
+        }
+
+      g_autoptr(GMappedFile) mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
+      if (!mapping)
+        {
+          soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+          goto out;
+        }
+      (void) close (fd); fd = -1;
+
+      /* Send caching headers */
+      g_autoptr(GDateTime) last_modified = g_date_time_new_from_unix_utc (stbuf.st_mtim.tv_sec);
+      if (last_modified != NULL)
+        {
+          g_autofree gchar *formatted = g_date_time_format (last_modified, "%a, %d %b %Y %H:%M:%S GMT");
+          soup_message_headers_append (msg->response_headers, "Last-Modified", formatted);
+        }
+
+      g_autofree gchar *etag = calculate_etag (mapping);
+      if (etag != NULL)
+        soup_message_headers_append (msg->response_headers, "ETag", etag);
       
       if (msg->method == SOUP_METHOD_GET)
         {
-          glnx_autofd int fd = -1;
-          g_autoptr(GMappedFile) mapping = NULL;
           gsize buffer_length, file_size;
           SoupRange *ranges;
           int ranges_length;
           gboolean have_ranges;
 
-          fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
-          if (fd < 0)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-
-          mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
-          if (!mapping)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-          (void) close (fd); fd = -1;
-
           file_size = g_mapped_file_get_length (mapping);
           have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length);
           if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL)
@@ -447,7 +466,50 @@ do_get (OtTrivialHttpd    *self,
           soup_message_headers_append (msg->response_headers,
                                        "Content-Length", length);
         }
-      soup_message_set_status (msg, SOUP_STATUS_OK);
+
+      /* Check client’s caching headers. */
+      const gchar *if_modified_since = soup_message_headers_get_one (msg->request_headers,
+                                                                     "If-Modified-Since");
+      const gchar *if_none_match = soup_message_headers_get_one (msg->request_headers,
+                                                                 "If-None-Match");
+
+      if (if_none_match != NULL && etag != NULL)
+        {
+          if (g_strcmp0 (etag, if_none_match) == 0)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED);
+              soup_message_body_truncate (msg->response_body);
+            }
+          else
+            {
+              soup_message_set_status (msg, SOUP_STATUS_OK);
+            }
+        }
+      else if (if_modified_since != NULL && last_modified != NULL)
+        {
+          SoupDate *if_modified_since_sd = soup_date_new_from_string (if_modified_since);
+          g_autoptr(GDateTime) if_modified_since_dt = NULL;
+
+          if (if_modified_since_sd != NULL)
+            if_modified_since_dt = g_date_time_new_from_unix_utc (soup_date_to_time_t (if_modified_since_sd));
+
+          if (if_modified_since_dt != NULL &&
+              g_date_time_compare (last_modified, if_modified_since_dt) <= 0)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_NOT_MODIFIED);
+              soup_message_body_truncate (msg->response_body);
+            }
+          else
+            {
+              soup_message_set_status (msg, SOUP_STATUS_OK);
+            }
+
+          g_clear_pointer (&if_modified_since_sd, soup_date_free);
+        }
+      else
+        {
+          soup_message_set_status (msg, SOUP_STATUS_OK);
+        }
     }
  out:
   {

src/ostree/ot-builtin-checkout.c

@@ -123,8 +123,6 @@ process_one_checkout (OstreeRepo           *repo,
                       GCancellable         *cancellable,
                       GError              **error)
 {
-  gboolean ret = FALSE;
-
   /* This strange code structure is to preserve testing
    * coverage of both `ostree_repo_checkout_tree` and
    * `ostree_repo_checkout_at` until such time as we have a more
@@ -145,33 +143,15 @@ process_one_checkout (OstreeRepo           *repo,
         checkout_options.mode = OSTREE_REPO_CHECKOUT_MODE_USER;
       /* Can't union these */
       if (opt_union && opt_union_add)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Cannot specify both --union and --union-add");
-          goto out;
-        }
+        return glnx_throw (error, "Cannot specify both --union and --union-add");
       if (opt_union && opt_union_identical)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Cannot specify both --union and --union-identical");
-          goto out;
-        }
+        return glnx_throw (error, "Cannot specify both --union and --union-identical");
       if (opt_union_add && opt_union_identical)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Cannot specify both --union-add and --union-identical ");
-          goto out;
-        }
+        return glnx_throw (error, "Cannot specify both --union-add and --union-identical");
       if (opt_require_hardlinks && opt_force_copy)
-        {
-          glnx_throw (error, "Cannot specify both --require-hardlinks and --force-copy");
-          goto out;
-        }
+        return glnx_throw (error, "Cannot specify both --require-hardlinks and --force-copy");
       if (opt_selinux_prefix && !opt_selinux_policy)
-        {
-          glnx_throw (error, "Cannot specify --selinux-prefix without --selinux-policy");
-          goto out;
-        }
+        return glnx_throw (error, "Cannot specify --selinux-prefix without --selinux-policy");
       else if (opt_union)
         checkout_options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES;
       else if (opt_union_add)
@@ -179,11 +159,7 @@ process_one_checkout (OstreeRepo           *repo,
       else if (opt_union_identical)
         {
           if (!opt_require_hardlinks)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "--union-identical requires --require-hardlinks");
-              goto out;
-            }
+            return glnx_throw (error, "--union-identical requires --require-hardlinks");
           checkout_options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL;
         }
       if (opt_whiteouts)
@@ -196,13 +172,10 @@ process_one_checkout (OstreeRepo           *repo,
         {
           glnx_autofd int rootfs_dfd = -1;
           if (!glnx_opendirat (AT_FDCWD, opt_selinux_policy, TRUE, &rootfs_dfd, error))
-            {
-              g_prefix_error (error, "selinux-policy: ");
-              goto out;
-            }
+            return glnx_prefix_error (error, "selinux-policy: ");
           policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error);
           if (!policy)
-            goto out;
+            return FALSE;
           checkout_options.sepolicy = policy;
           checkout_options.sepolicy_prefix = opt_selinux_prefix;
         }
@@ -213,7 +186,7 @@ process_one_checkout (OstreeRepo           *repo,
         {
           if (!ot_parse_file_by_line (opt_skiplist_file, handle_skiplist_line, skip_list,
                                       cancellable, error))
-            goto out;
+            return FALSE;
           checkout_options.filter = checkout_filter;
           checkout_options.filter_user_data = skip_list;
         }
@@ -227,25 +200,25 @@ process_one_checkout (OstreeRepo           *repo,
                                     AT_FDCWD, destination,
                                     resolved_commit,
                                     cancellable, error))
-        goto out;
+        return FALSE;
     }
   else
     {
       GError *tmp_error = NULL;
       g_autoptr(GFile) root = NULL;
-      g_autoptr(GFile) subtree = NULL;
-      g_autoptr(GFileInfo) file_info = NULL;
       g_autoptr(GFile) destination_file = g_file_new_for_path (destination);
 
       if (!ostree_repo_read_commit (repo, resolved_commit, &root, NULL, cancellable, error))
-        goto out;
+        return FALSE;
 
+      g_autoptr(GFile) subtree = NULL;
       if (subpath)
         subtree = g_file_resolve_relative_path (root, subpath);
       else
         subtree = g_object_ref (root);
 
-      file_info = g_file_query_info (subtree, OSTREE_GIO_FAST_QUERYINFO,
+      g_autoptr(GFileInfo) file_info
+        = g_file_query_info (subtree, OSTREE_GIO_FAST_QUERYINFO,
                                      G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
                                      cancellable, &tmp_error);
       if (!file_info)
@@ -254,13 +227,14 @@ process_one_checkout (OstreeRepo           *repo,
               && g_error_matches (tmp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
             {
               g_clear_error (&tmp_error);
-              ret = TRUE;
+              /* Note early return */
+              return TRUE;
             }
           else
             {
               g_propagate_error (error, tmp_error);
+              return FALSE;
             }
-          goto out;
         }
 
       if (!ostree_repo_checkout_tree (repo, opt_user_mode ? OSTREE_REPO_CHECKOUT_MODE_USER : 0,
@@ -268,12 +242,10 @@ process_one_checkout (OstreeRepo           *repo,
                                       destination_file,
                                       OSTREE_REPO_FILE (subtree), file_info,
                                       cancellable, error))
-        goto out;
+        return FALSE;
     }
 
-  ret = TRUE;
- out:
-  return ret;
+  return TRUE;
 }
 
 static gboolean
@@ -352,56 +324,46 @@ process_many_checkouts (OstreeRepo         *repo,
 gboolean
 ostree_builtin_checkout (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = NULL;
+  g_autoptr(GOptionContext) context = g_option_context_new ("COMMIT [DESTINATION]");
   g_autoptr(OstreeRepo) repo = NULL;
-  gboolean ret = FALSE;
-  const char *commit;
-  const char *destination;
-  g_autofree char *resolved_commit = NULL;
-
-  context = g_option_context_new ("COMMIT [DESTINATION]");
-
   if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
-    goto out;
+    return FALSE;
 
   if (opt_disable_fsync)
     ostree_repo_set_disable_fsync (repo, TRUE);
 
   if (argc < 2)
     {
-      gchar *help = g_option_context_get_help (context, TRUE, NULL);
+      g_autofree char *help = g_option_context_get_help (context, TRUE, NULL);
       g_printerr ("%s\n", help);
-      g_free (help);
-      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "COMMIT must be specified");
-      goto out;
+      return glnx_throw (error, "COMMIT must be specified");
     }
 
   if (opt_from_stdin || opt_from_file)
     {
-      destination = argv[1];
+      const char *destination = argv[1];
 
       if (!process_many_checkouts (repo, destination, cancellable, error))
-        goto out;
+        return FALSE;
     }
   else
     {
-      commit = argv[1];
+      const char *commit = argv[1];
+      const char *destination;
       if (argc < 3)
         destination = commit;
       else
         destination = argv[2];
 
+      g_autofree char *resolved_commit = NULL;
       if (!ostree_repo_resolve_rev (repo, commit, FALSE, &resolved_commit, error))
-        goto out;
+        return FALSE;
 
       if (!process_one_checkout (repo, resolved_commit, opt_subpath,
                                  destination,
                                  cancellable, error))
-        goto out;
+        return FALSE;
     }
 
-  ret = TRUE;
- out:
-  return ret;
+  return TRUE;
 }

src/ostree/ot-builtin-static-delta.c

@@ -53,6 +53,8 @@ BUILTINPROTO(delete);
 BUILTINPROTO(generate);
 BUILTINPROTO(apply_offline);
 BUILTINPROTO(verify);
+BUILTINPROTO(indexes);
+BUILTINPROTO(reindex);
 
 #undef BUILTINPROTO
 
@@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = {
   { "verify", OSTREE_BUILTIN_FLAG_NONE,
     ot_static_delta_builtin_verify,
     "Verify static delta signatures" },
+  { "indexes", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_indexes,
+    "List static delta indexes" },
+  { "reindex", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_reindex,
+    "Regenerate static delta indexes" },
   { NULL, 0, NULL, NULL }
 };
 
@@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = {
   { NULL }
 };
 
+static GOptionEntry indexes_options[] = {
+  { NULL }
+};
+
+static GOptionEntry reindex_options[] = {
+  { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" },
+  { NULL }
+};
+
 static void
 static_delta_usage (char    **argv,
                     gboolean  is_error)
@@ -176,6 +193,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in
   return TRUE;
 }
 
+static gboolean
+ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(OstreeRepo) repo = NULL;
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+  if (!ostree_option_context_parse (context, indexes_options, &argc, &argv,
+                                    invocation, &repo, cancellable, error))
+    return FALSE;
+
+  g_autoptr(GPtrArray) indexes = NULL;
+  if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error))
+    return FALSE;
+
+  if (indexes->len == 0)
+    g_print ("(No static deltas indexes)\n");
+  else
+    {
+      for (guint i = 0; i < indexes->len; i++)
+        g_print ("%s\n", (char*)indexes->pdata[i]);
+    }
+
+  return TRUE;
+}
+
+static gboolean
+ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
+{
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+
+  g_autoptr(OstreeRepo) repo = NULL;
+  if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error))
+    return FALSE;
+
+  if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+
 static gboolean
 ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {

src/switchroot/ostree-prepare-root.c

@@ -152,8 +152,8 @@ resolve_deploy_path (const char * root_mountpoint)
                    "MESSAGE_ID=" SD_ID128_FORMAT_STR,
                    SD_ID128_FORMAT_VAL(OSTREE_PREPARE_ROOT_DEPLOYMENT_MSG),
                    "DEPLOYMENT_PATH=%s", resolved_path,
-                   "DEPLOYMENT_DEVICE=%u", stbuf.st_dev,
-                   "DEPLOYMENT_INODE=%u", stbuf.st_ino,
+                   "DEPLOYMENT_DEVICE=%" PRIu64, (uint64_t) stbuf.st_dev,
+                   "DEPLOYMENT_INODE=%" PRIu64, (uint64_t) stbuf.st_ino,
                    NULL);
 #endif
   return deploy_path;