New upstream version 2017.13

src/libostree/libostree-devel.sym

@@ -18,8 +18,8 @@
 ***/
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
-LIBOSTREE_2017.13 {
-} LIBOSTREE_2017.12;
+LIBOSTREE_2017.14 {
+} LIBOSTREE_2017.13;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste
@@ -27,6 +27,6 @@ LIBOSTREE_2017.13 {
  * with whatever the next version with new symbols will be.
 LIBOSTREE_2017.$NEWVERSION {
 global:
-	someostree_symbol_deleteme;
+  someostree_symbol_deleteme;
 } LIBOSTREE_2017.$LASTSTABLE;
 */

src/libostree/libostree-experimental.sym

@@ -82,3 +82,10 @@ LIBOSTREE_2017.12_EXPERIMENTAL {
 global:
   ostree_repo_resolve_collection_ref;
 } LIBOSTREE_2017.8_EXPERIMENTAL;
+
+LIBOSTREE_2017.13_EXPERIMENTAL {
+global:
+  ostree_repo_finder_override_add_uri;
+  ostree_repo_finder_override_get_type;
+  ostree_repo_finder_override_new;
+} LIBOSTREE_2017.12_EXPERIMENTAL;

src/libostree/libostree-released.sym

@@ -436,6 +436,12 @@ global:
   ostree_repo_hash;
 } LIBOSTREE_2017.11;
 
+LIBOSTREE_2017.13 {
+global:
+  ostree_checksum_file_at;
+  ostree_repo_checkout_at_options_set_devino;
+} LIBOSTREE_2017.12;
+
 /* NOTE: Only add more content here in release commits!  See the
  * comments at the top of this file.
  */

src/libostree/ostree-autocleanups.h

@@ -66,6 +66,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinder, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderAvahi, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderOverride, g_object_unref)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free)
 G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL)
 #endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */

src/libostree/ostree-bloom.c

@@ -76,7 +76,7 @@
 struct _OstreeBloom
 {
   guint ref_count;
-  gsize n_bytes;
+  gsize n_bytes;  /* 0 < n_bytes <= G_MAXSIZE / 8 */
   gboolean is_mutable;  /* determines which of [im]mutable_bytes is accessed */
   union
     {
@@ -117,6 +117,7 @@ ostree_bloom_new (gsize               n_bytes,
   g_autoptr(OstreeBloom) bloom = NULL;
 
   g_return_val_if_fail (n_bytes > 0, NULL);
+  g_return_val_if_fail (n_bytes <= G_MAXSIZE / 8, NULL);
   g_return_val_if_fail (k > 0, NULL);
   g_return_val_if_fail (hash_func != NULL, NULL);
 
@@ -159,6 +160,7 @@ ostree_bloom_new_from_bytes (GBytes              *bytes,
 
   g_return_val_if_fail (bytes != NULL, NULL);
   g_return_val_if_fail (g_bytes_get_size (bytes) > 0, NULL);
+  g_return_val_if_fail (g_bytes_get_size (bytes) <= G_MAXSIZE / 8, NULL);
   g_return_val_if_fail (k > 0, NULL);
   g_return_val_if_fail (hash_func != NULL, NULL);
 

src/libostree/ostree-bootloader-grub2.c

@@ -416,7 +416,7 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader      *bootloader,
     }
 
   /* Now let's fdatasync() for the new file */
-  { glnx_fd_close int new_config_fd = -1;
+  { glnx_autofd int new_config_fd = -1;
     if (!glnx_openat_rdonly (AT_FDCWD, gs_file_get_path_cached (new_config_path), TRUE, &new_config_fd, error))
       return FALSE;
 

src/libostree/ostree-bootloader-syslinux.c

@@ -25,12 +25,13 @@
 
 #include <string.h>
 
+static const char syslinux_config_path[] = "boot/syslinux/syslinux.cfg";
+
 struct _OstreeBootloaderSyslinux
 {
   GObject       parent_instance;
 
   OstreeSysroot  *sysroot;
-  GFile          *config_path;
 };
 
 typedef GObjectClass OstreeBootloaderSyslinuxClass;
@@ -46,8 +47,11 @@ _ostree_bootloader_syslinux_query (OstreeBootloader *bootloader,
                                    GError          **error)
 {
   OstreeBootloaderSyslinux *self = OSTREE_BOOTLOADER_SYSLINUX (bootloader);
+  struct stat stbuf;
 
-  *out_is_active = g_file_query_file_type (self->config_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == G_FILE_TYPE_SYMBOLIC_LINK;
+  if (!glnx_fstatat_allow_noent (self->sysroot->sysroot_fd, syslinux_config_path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  *out_is_active = (errno == 0);
   return TRUE;
 }
 
@@ -107,12 +111,12 @@ _ostree_bootloader_syslinux_write_config (OstreeBootloader          *bootloader,
 {
   OstreeBootloaderSyslinux *self = OSTREE_BOOTLOADER_SYSLINUX (bootloader);
 
-  g_autoptr(GFile) new_config_path =
-    ot_gfile_resolve_path_printf (self->sysroot->path, "boot/loader.%d/syslinux.cfg", bootversion);
+  g_autofree char *new_config_path =
+    g_strdup_printf ("boot/loader.%d/syslinux.cfg", bootversion);
 
   /* This should follow the symbolic link to the current bootversion. */
   g_autofree char *config_contents =
-    glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (self->config_path), NULL,
+    glnx_file_get_contents_utf8_at (self->sysroot->sysroot_fd, syslinux_config_path, NULL,
                                     cancellable, error);
   if (!config_contents)
     return FALSE;
@@ -206,12 +210,10 @@ _ostree_bootloader_syslinux_write_config (OstreeBootloader          *bootloader,
     return FALSE;
 
   g_autofree char *new_config_contents = _ostree_sysroot_join_lines (new_lines);
-  g_autoptr(GBytes) new_config_contents_bytes =
-    g_bytes_new_static (new_config_contents,
-                        strlen (new_config_contents));
-
-  if (!ot_gfile_replace_contents_fsync (new_config_path, new_config_contents_bytes,
-                                        cancellable, error))
+  if (!glnx_file_replace_contents_at (self->sysroot->sysroot_fd, new_config_path,
+                                      (guint8*)new_config_contents, strlen (new_config_contents),
+                                      GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
     return FALSE;
 
   return TRUE;
@@ -223,7 +225,6 @@ _ostree_bootloader_syslinux_finalize (GObject *object)
   OstreeBootloaderSyslinux *self = OSTREE_BOOTLOADER_SYSLINUX (object);
 
   g_clear_object (&self->sysroot);
-  g_clear_object (&self->config_path);
 
   G_OBJECT_CLASS (_ostree_bootloader_syslinux_parent_class)->finalize (object);
 }
@@ -254,6 +255,5 @@ _ostree_bootloader_syslinux_new (OstreeSysroot *sysroot)
 {
   OstreeBootloaderSyslinux *self = g_object_new (OSTREE_TYPE_BOOTLOADER_SYSLINUX, NULL);
   self->sysroot = g_object_ref (sysroot);
-  self->config_path = g_file_resolve_relative_path (self->sysroot->path, "boot/syslinux/syslinux.cfg");
   return self;
 }

src/libostree/ostree-bootloader-uboot.c

@@ -29,12 +29,13 @@
 
 #include <string.h>
 
+static const char uboot_config_path[] = "boot/loader/uEnv.txt";
+
 struct _OstreeBootloaderUboot
 {
   GObject       parent_instance;
 
   OstreeSysroot  *sysroot;
-  GFile          *config_path;
 };
 
 typedef GObjectClass OstreeBootloaderUbootClass;
@@ -50,8 +51,11 @@ _ostree_bootloader_uboot_query (OstreeBootloader *bootloader,
                                 GError          **error) 
 {
   OstreeBootloaderUboot *self = OSTREE_BOOTLOADER_UBOOT (bootloader);
+  struct stat stbuf;
 
-  *out_is_active = g_file_query_file_type (self->config_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == G_FILE_TYPE_REGULAR;
+  if (!glnx_fstatat_allow_noent (self->sysroot->sysroot_fd, uboot_config_path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  *out_is_active = (errno == 0);
   return TRUE;
 }
 
@@ -69,8 +73,8 @@ append_system_uenv (OstreeBootloaderUboot   *self,
                     GCancellable            *cancellable,
                     GError                 **error)
 {
-  glnx_fd_close int uenv_fd = -1;
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = NULL;
+  glnx_autofd int uenv_fd = -1;
+  g_autoptr(OstreeKernelArgs) kargs = NULL;
   const char *uenv_path = NULL;
   const char *ostree_arg = NULL;
 
@@ -156,36 +160,26 @@ _ostree_bootloader_uboot_write_config (OstreeBootloader          *bootloader,
                                   GError               **error)
 {
   OstreeBootloaderUboot *self = OSTREE_BOOTLOADER_UBOOT (bootloader);
-  g_autoptr(GFile) new_config_path = NULL;
-  g_autofree char *config_contents = NULL;
-  g_autofree char *new_config_contents = NULL;
-  g_autoptr(GPtrArray) new_lines = NULL;
 
   /* This should follow the symbolic link to the current bootversion. */
-  config_contents = glnx_file_get_contents_utf8_at (AT_FDCWD, gs_file_get_path_cached (self->config_path), NULL,
-                                                    cancellable, error);
+  g_autofree char *config_contents =
+    glnx_file_get_contents_utf8_at (self->sysroot->sysroot_fd, uboot_config_path, NULL,
+                                    cancellable, error);
   if (!config_contents)
     return FALSE;
 
-  new_config_path = ot_gfile_resolve_path_printf (self->sysroot->path, "boot/loader.%d/uEnv.txt",
-                                                      bootversion);
-
-  new_lines = g_ptr_array_new_with_free_func (g_free);
-
+  g_autoptr(GPtrArray) new_lines = g_ptr_array_new_with_free_func (g_free);
   if (!create_config_from_boot_loader_entries (self, bootversion, new_lines,
                                                cancellable, error))
     return FALSE;
 
-  new_config_contents = _ostree_sysroot_join_lines (new_lines);
-  {
-    g_autoptr(GBytes) new_config_contents_bytes =
-      g_bytes_new_static (new_config_contents,
-                          strlen (new_config_contents));
-
-    if (!ot_gfile_replace_contents_fsync (new_config_path, new_config_contents_bytes,
-                                          cancellable, error))
-      return FALSE;
-  }
+  g_autofree char *new_config_path = g_strdup_printf ("boot/loader.%d/uEnv.txt", bootversion);
+  g_autofree char *new_config_contents = _ostree_sysroot_join_lines (new_lines);
+  if (!glnx_file_replace_contents_at (self->sysroot->sysroot_fd, new_config_path,
+                                      (guint8*)new_config_contents, strlen (new_config_contents),
+                                      GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
+    return FALSE;
 
   return TRUE;
 }
@@ -196,7 +190,6 @@ _ostree_bootloader_uboot_finalize (GObject *object)
   OstreeBootloaderUboot *self = OSTREE_BOOTLOADER_UBOOT (object);
 
   g_clear_object (&self->sysroot);
-  g_clear_object (&self->config_path);
 
   G_OBJECT_CLASS (_ostree_bootloader_uboot_parent_class)->finalize (object);
 }
@@ -227,6 +220,5 @@ _ostree_bootloader_uboot_new (OstreeSysroot *sysroot)
 {
   OstreeBootloaderUboot *self = g_object_new (OSTREE_TYPE_BOOTLOADER_UBOOT, NULL);
   self->sysroot = g_object_ref (sysroot);
-  self->config_path = g_file_resolve_relative_path (self->sysroot->path, "boot/loader/uEnv.txt");
   return self;
 }

src/libostree/ostree-core-private.h

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "ostree-core.h"
+#include "otutil.h"
 #include <sys/stat.h>
 
 G_BEGIN_DECLS
@@ -67,19 +68,11 @@ G_BEGIN_DECLS
 #define _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT G_VARIANT_TYPE ("(tuuuusa(ayay))")
 
 
-GVariant *_ostree_file_header_new (GFileInfo         *file_info,
-                                   GVariant          *xattrs);
+GBytes *_ostree_file_header_new (GFileInfo         *file_info,
+                                 GVariant          *xattrs);
 
-GVariant *_ostree_zlib_file_header_new (GFileInfo         *file_info,
-                                        GVariant          *xattrs);
-
-gboolean _ostree_write_variant_with_size (GOutputStream      *output,
-                                          GVariant           *variant,
-                                          guint64             alignment_offset,
-                                          gsize              *out_bytes_written,
-                                          GChecksum          *checksum,
-                                          GCancellable       *cancellable,
-                                          GError            **error);
+GBytes *_ostree_zlib_file_header_new (GFileInfo         *file_info,
+                                      GVariant          *xattrs);
 
 gboolean
 _ostree_make_temporary_symlink_at (int             tmp_dirfd,
@@ -90,6 +83,7 @@ _ostree_make_temporary_symlink_at (int             tmp_dirfd,
 
 GFileInfo * _ostree_stbuf_to_gfileinfo (const struct stat *stbuf);
 gboolean _ostree_gfileinfo_equal (GFileInfo *a, GFileInfo *b);
+gboolean _ostree_stbuf_equal (struct stat *stbuf_a, struct stat *stbuf_b);
 GFileInfo * _ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid);
 
 static inline void
@@ -134,6 +128,11 @@ static inline char * _ostree_get_commitpartial_path (const char *checksum)
   return g_strconcat ("state/", checksum, ".commitpartial", NULL);
 }
 
+gboolean
+_ostree_validate_ref_fragment (const char *fragment,
+                               GError    **error);
+
+
 gboolean
 _ostree_validate_bareuseronly_mode (guint32     mode,
                                     const char *checksum,
@@ -147,6 +146,12 @@ _ostree_validate_bareuseronly_mode_finfo (GFileInfo  *finfo,
   return _ostree_validate_bareuseronly_mode (content_mode, checksum, error);
 }
 
+gboolean
+_ostree_compare_object_checksum (OstreeObjectType objtype,
+                                 const char      *expected,
+                                 const char      *actual,
+                                 GError         **error);
+
 gboolean
 _ostree_parse_delta_name (const char  *delta_name,
                           char        **out_from,
@@ -218,6 +223,9 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderConfig, g_object_unref)
 
 #include "ostree-repo-finder-mount.h"
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderMount, g_object_unref)
+
+#include "ostree-repo-finder-override.h"
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderOverride, g_object_unref)
 #endif
 
 G_END_DECLS

src/libostree/ostree-core.c

@@ -39,6 +39,8 @@ G_STATIC_ASSERT(OSTREE_REPO_MODE_ARCHIVE == OSTREE_REPO_MODE_ARCHIVE_Z2);
 G_STATIC_ASSERT(OSTREE_REPO_MODE_BARE_USER == 2);
 G_STATIC_ASSERT(OSTREE_REPO_MODE_BARE_USER_ONLY == 3);
 
+static GBytes *variant_to_lenprefixed_buffer (GVariant *variant);
+
 #define ALIGN_VALUE(this, boundary) \
   (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1)))
 
@@ -140,7 +142,10 @@ ostree_validate_checksum_string (const char *sha256,
   return ostree_validate_structureof_checksum_string (sha256, error);
 }
 
-#define OSTREE_REF_FRAGMENT_REGEXP "[-._\\w\\d]+"
+/* This used to allow leading - and ., but was changed in
+ * https://github.com/ostreedev/ostree/pull/1286
+ */
+#define OSTREE_REF_FRAGMENT_REGEXP "[\\w\\d][-._\\w\\d]*"
 #define OSTREE_REF_REGEXP "(?:" OSTREE_REF_FRAGMENT_REGEXP "/)*" OSTREE_REF_FRAGMENT_REGEXP
 #define OSTREE_REMOTE_NAME_REGEXP OSTREE_REF_FRAGMENT_REGEXP
 
@@ -194,6 +199,26 @@ ostree_parse_refspec (const char   *refspec,
   return TRUE;
 }
 
+gboolean
+_ostree_validate_ref_fragment (const char *fragment,
+                               GError    **error)
+{
+  static GRegex *regex;
+  static gsize regex_initialized;
+  if (g_once_init_enter (&regex_initialized))
+    {
+      regex = g_regex_new ("^" OSTREE_REF_FRAGMENT_REGEXP "$", 0, 0, NULL);
+      g_assert (regex);
+      g_once_init_leave (&regex_initialized, 1);
+    }
+
+  g_autoptr(GMatchInfo) match = NULL;
+  if (!g_regex_match (regex, fragment, 0, &match))
+    return glnx_throw (error, "Invalid ref fragment '%s'", fragment);
+
+  return TRUE;
+}
+
 /**
  * ostree_validate_rev:
  * @rev: A revision string
@@ -283,192 +308,103 @@ ostree_validate_collection_id (const char *collection_id, GError **error)
   return TRUE;
 }
 
-GVariant *
+/* The file header is part of the "object stream" format
+ * that's not compressed.  It's comprised of uid,gid,mode,
+ * and possibly symlink targets from @file_info, as well
+ * as @xattrs (which if NULL, is taken to be the empty set).
+ */
+GBytes *
 _ostree_file_header_new (GFileInfo         *file_info,
                          GVariant          *xattrs)
 {
-  guint32 uid;
-  guint32 gid;
-  guint32 mode;
+
+  guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+  guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+  guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+
   const char *symlink_target;
-  GVariant *ret;
-  g_autoptr(GVariant) tmp_xattrs = NULL;
-
-  uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
-  gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
-  mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
-
   if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK)
     symlink_target = g_file_info_get_symlink_target (file_info);
   else
     symlink_target = "";
 
+  g_autoptr(GVariant) tmp_xattrs = NULL;
   if (xattrs == NULL)
     tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
 
-  ret = g_variant_new ("(uuuus@a(ayay))", GUINT32_TO_BE (uid),
-                       GUINT32_TO_BE (gid), GUINT32_TO_BE (mode), 0,
-                       symlink_target, xattrs ? xattrs : tmp_xattrs);
-  g_variant_ref_sink (ret);
-  return ret;
+  g_autoptr(GVariant) ret = g_variant_new ("(uuuus@a(ayay))", GUINT32_TO_BE (uid),
+                                           GUINT32_TO_BE (gid), GUINT32_TO_BE (mode), 0,
+                                           symlink_target, xattrs ?: tmp_xattrs);
+  return variant_to_lenprefixed_buffer (g_variant_ref_sink (ret));
 }
 
-/*
- * ostree_zlib_file_header_new:
- * @file_info: a #GFileInfo
- * @xattrs: (allow-none): Optional extended attribute array
- *
- * Returns: (transfer full): A new #GVariant containing file header for an archive repository
+/* Like _ostree_file_header_new(), but used for the compressed format in archive
+ * repositories. This format hence lives on disk; normally the uncompressed
+ * stream format doesn't. Instead for "bare" repositories, the file data is
+ * stored directly, or for the special case of bare-user repositories, as a
+ * user.ostreemeta xattr.
  */
-GVariant *
+GBytes *
 _ostree_zlib_file_header_new (GFileInfo         *file_info,
                               GVariant          *xattrs)
 {
-  guint64 size;
-  guint32 uid;
-  guint32 gid;
-  guint32 mode;
+  guint64 size = g_file_info_get_size (file_info);
+  guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+  guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+  guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+
   const char *symlink_target;
-  GVariant *ret;
-  g_autoptr(GVariant) tmp_xattrs = NULL;
-
-  size = g_file_info_get_size (file_info);
-  uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
-  gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
-  mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
-
   if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK)
     symlink_target = g_file_info_get_symlink_target (file_info);
   else
     symlink_target = "";
 
+  g_autoptr(GVariant) tmp_xattrs = NULL;
   if (xattrs == NULL)
     tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
 
-  ret = g_variant_new ("(tuuuus@a(ayay))",
-                       GUINT64_TO_BE (size), GUINT32_TO_BE (uid),
-                       GUINT32_TO_BE (gid), GUINT32_TO_BE (mode), 0,
-                       symlink_target, xattrs ? xattrs : tmp_xattrs);
-  g_variant_ref_sink (ret);
-  return ret;
+  g_autoptr(GVariant) ret = g_variant_new ("(tuuuus@a(ayay))",
+                                           GUINT64_TO_BE (size), GUINT32_TO_BE (uid),
+                                           GUINT32_TO_BE (gid), GUINT32_TO_BE (mode), 0,
+                                           symlink_target, xattrs ?: tmp_xattrs);
+  return variant_to_lenprefixed_buffer (g_variant_ref_sink (ret));
 }
 
-static gboolean
-write_padding (GOutputStream    *output,
-               guint             alignment,
-               gsize             offset,
-               gsize            *out_bytes_written,
-               GChecksum        *checksum,
-               GCancellable     *cancellable,
-               GError          **error)
+/* Serialize a variant to a buffer prefixed with its length. The variant will
+ * have an 8-byte alignment so it can be safely used with `mmap()`.
+ */
+static GBytes *
+variant_to_lenprefixed_buffer (GVariant *variant)
 {
-  guint bits;
-  guint padding_len;
-  guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+  /* This string is really a binary memory buffer */
+  g_autoptr(GString) buf = g_string_new (NULL);
+  /* Write variant size */
+  const guint64 variant_size = g_variant_get_size (variant);
+  g_assert (variant_size < G_MAXUINT32);
+  const guint32 variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size);
 
+  g_string_append_len (buf, (char*)&variant_size_u32_be, sizeof (variant_size_u32_be));
+  const gsize alignment_offset = sizeof (variant_size_u32_be);
+
+  /* Write NULs for alignment. At the moment this is a constant 4 bytes (i.e.
+   * align to 8, since the length is 4 bytes). For now, I decided to keep some
+   * of the (now legacy) more generic logic here in case we want to revive it
+   * later.
+   */
+  const guint alignment = 8;
+  const guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+  guint bits;
   if (alignment == 8)
-    bits = ((offset) & 7);
+    bits = alignment_offset & 7; /* mod 8 */
   else
-    bits = ((offset) & 3);
+    bits = alignment_offset & 3; /* mod 4 */
+  const guint padding_len = alignment - bits;
 
   if (bits > 0)
-    {
-      padding_len = alignment - bits;
-      if (!ot_gio_write_update_checksum (output, (guchar*)padding_nuls, padding_len,
-                                         out_bytes_written, checksum,
-                                         cancellable, error))
-        return FALSE;
-    }
+    g_string_append_len (buf, (char*)padding_nuls, padding_len);
 
-  return TRUE;
-}
-
-/*
- * _ostree_write_variant_with_size:
- * @output: Stream
- * @variant: A variant
- * @alignment_offset: Used to determine whether or not we should write padding bytes
- * @out_bytes_written: (out): Number of bytes written
- * @checksum: (allow-none): If provided, update with written data
- * @cancellable: Cancellable
- * @error: Error
- *
- * Use this function for serializing a chain of 1 or more variants
- * into a stream; the @alignment_offset parameter is used to ensure
- * that each variant begins on an 8-byte alignment so it can be safely
- * accessed.
- */
-gboolean
-_ostree_write_variant_with_size (GOutputStream      *output,
-                                 GVariant           *variant,
-                                 guint64             alignment_offset,
-                                 gsize              *out_bytes_written,
-                                 GChecksum          *checksum,
-                                 GCancellable       *cancellable,
-                                 GError            **error)
-{
-  guint64 variant_size;
-  guint32 variant_size_u32_be;
-  gsize bytes_written;
-  gsize ret_bytes_written = 0;
-
-  /* Write variant size */
-  variant_size = g_variant_get_size (variant);
-  g_assert (variant_size < G_MAXUINT32);
-  variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size);
-
-  bytes_written = 0;
-  if (!ot_gio_write_update_checksum (output, &variant_size_u32_be, 4,
-                                     &bytes_written, checksum,
-                                     cancellable, error))
-    return FALSE;
-  ret_bytes_written += bytes_written;
-  alignment_offset += bytes_written;
-
-  bytes_written = 0;
-  /* Pad to offset of 8, write variant */
-  if (!write_padding (output, 8, alignment_offset, &bytes_written, checksum,
-                      cancellable, error))
-    return FALSE;
-  ret_bytes_written += bytes_written;
-
-  bytes_written = 0;
-  if (!ot_gio_write_update_checksum (output, g_variant_get_data (variant),
-                                     variant_size, &bytes_written, checksum,
-                                     cancellable, error))
-    return FALSE;
-  ret_bytes_written += bytes_written;
-
-  if (out_bytes_written)
-    *out_bytes_written = ret_bytes_written;
-  return TRUE;
-}
-
-/*
- * write_file_header_update_checksum:
- * @out: Stream
- * @variant: A variant, should be a file header
- * @checksum: (allow-none): If provided, update with written data
- * @cancellable: Cancellable
- * @error: Error
- *
- * Write a file header variant to the provided @out stream, optionally
- * updating @checksum.
- */
-static gboolean
-write_file_header_update_checksum (GOutputStream         *out,
-                                   GVariant              *header,
-                                   GChecksum             *checksum,
-                                   GCancellable          *cancellable,
-                                   GError               **error)
-{
-  gsize bytes_written;
-
-  if (!_ostree_write_variant_with_size (out, header, 0, &bytes_written, checksum,
-                                        cancellable, error))
-    return FALSE;
-
-  return TRUE;
+  g_string_append_len (buf, (char*)g_variant_get_data (variant), g_variant_get_size (variant));
+  return g_string_free_to_bytes (g_steal_pointer (&buf));
 }
 
 /*
@@ -476,54 +412,38 @@ write_file_header_update_checksum (GOutputStream         *out,
  * @file_header: A file header
  * @input: File raw content stream
  * @out_input: (out): Serialized object stream
- * @out_header_size: (out): Length of the header
  * @cancellable: Cancellable
  * @error: Error
  *
  * Combines @file_header and @input into a single stream.
  */
 static gboolean
-header_and_input_to_stream (GVariant           *file_header,
+header_and_input_to_stream (GBytes             *file_header,
                             GInputStream       *input,
                             GInputStream      **out_input,
-                            guint64            *out_header_size,
                             GCancellable       *cancellable,
                             GError            **error)
 {
-  gpointer header_data;
-  gsize header_size;
-  g_autoptr(GInputStream) ret_input = NULL;
-  g_autoptr(GPtrArray) streams = NULL;
-  g_autoptr(GOutputStream) header_out_stream = NULL;
-  g_autoptr(GInputStream) header_in_stream = NULL;
+  /* Our result stream chain */
+  g_autoptr(GPtrArray) streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
 
-  header_out_stream = g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-
-  if (!_ostree_write_variant_with_size (header_out_stream, file_header, 0, NULL, NULL,
-                                        cancellable, error))
-    return FALSE;
-
-  if (!g_output_stream_close (header_out_stream, cancellable, error))
-    return FALSE;
-
-  header_size = g_memory_output_stream_get_data_size ((GMemoryOutputStream*) header_out_stream);
-  header_data = g_memory_output_stream_steal_data ((GMemoryOutputStream*) header_out_stream);
-  header_in_stream = g_memory_input_stream_new_from_data (header_data, header_size, g_free);
-
-  streams = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
+  /* Append the header to the chain */
+  g_autoptr(GInputStream) header_in_stream = g_memory_input_stream_new_from_bytes (file_header);
 
   g_ptr_array_add (streams, g_object_ref (header_in_stream));
+
+  /* And if we have an input stream, append that */
   if (input)
     g_ptr_array_add (streams, g_object_ref (input));
 
-  ret_input = (GInputStream*)ostree_chain_input_stream_new (streams);
+  /* Return the result stream */
+  g_autoptr(GInputStream) ret_input = (GInputStream*)ostree_chain_input_stream_new (streams);
   ot_transfer_out_value (out_input, &ret_input);
-  if (out_header_size)
-    *out_header_size = header_size;
 
   return TRUE;
 }
 
+/* Convert file metadata + file content into an archive-format stream. */
 gboolean
 _ostree_raw_file_to_archive_stream (GInputStream       *input,
                                     GFileInfo          *file_info,
@@ -533,21 +453,17 @@ _ostree_raw_file_to_archive_stream (GInputStream       *input,
                                     GCancellable       *cancellable,
                                     GError            **error)
 {
-  g_autoptr(GVariant) file_header = NULL;
   g_autoptr(GInputStream) zlib_input = NULL;
-
-  file_header = _ostree_zlib_file_header_new (file_info, xattrs);
   if (input != NULL)
     {
-      g_autoptr(GConverter) zlib_compressor = NULL;
-
-      zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level));
+      g_autoptr(GConverter) zlib_compressor =
+        G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level));
       zlib_input = g_converter_input_stream_new (input, zlib_compressor);
     }
+  g_autoptr(GBytes) file_header = _ostree_zlib_file_header_new (file_info, xattrs);
   return header_and_input_to_stream (file_header,
                                      zlib_input,
                                      out_input,
-                                     NULL,
                                      cancellable,
                                      error);
 }
@@ -640,19 +556,15 @@ ostree_raw_file_to_content_stream (GInputStream       *input,
                                    GCancellable       *cancellable,
                                    GError            **error)
 {
-  g_autoptr(GVariant) file_header = NULL;
-  guint64 header_size;
-
-  file_header = _ostree_file_header_new (file_info, xattrs);
+  g_autoptr(GBytes) file_header = _ostree_file_header_new (file_info, xattrs);
   if (!header_and_input_to_stream (file_header,
                                    input,
                                    out_input,
-                                   &header_size,
                                    cancellable,
                                    error))
     return FALSE;
   if (out_length)
-    *out_length = header_size + g_file_info_get_size (file_info);
+    *out_length = g_bytes_get_size (file_header) + g_file_info_get_size (file_info);
   return TRUE;
 }
 
@@ -781,7 +693,7 @@ ostree_content_file_parse_at (gboolean                compressed,
                               GCancellable           *cancellable,
                               GError                **error)
 {
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   if (!glnx_openat_rdonly (parent_dfd, path, TRUE, &fd, error))
     return FALSE;
 
@@ -859,37 +771,35 @@ ostree_checksum_file_from_input (GFileInfo        *file_info,
                                  GError          **error)
 {
 
-  g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256);
+  g_auto(OtChecksum) checksum = { 0, };
+  ot_checksum_init (&checksum);
 
   if (OSTREE_OBJECT_TYPE_IS_META (objtype))
     {
-      if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+      if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error))
         return FALSE;
     }
   else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
     {
       g_autoptr(GVariant) dirmeta = ostree_create_directory_metadata (file_info, xattrs);
-      g_checksum_update (checksum, g_variant_get_data (dirmeta),
-                         g_variant_get_size (dirmeta));
+      ot_checksum_update (&checksum, g_variant_get_data (dirmeta),
+                          g_variant_get_size (dirmeta));
     }
   else
     {
-      g_autoptr(GVariant) file_header = NULL;
+      g_autoptr(GBytes) file_header = _ostree_file_header_new (file_info, xattrs);
 
-      file_header = _ostree_file_header_new (file_info, xattrs);
-
-      if (!write_file_header_update_checksum (NULL, file_header, checksum,
-                                              cancellable, error))
-        return FALSE;
+      ot_checksum_update_bytes (&checksum, file_header);
 
       if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
         {
-          if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+          if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error))
             return FALSE;
         }
     }
 
-  *out_csum = ot_csum_from_gchecksum (checksum);
+  *out_csum = g_malloc (OSTREE_SHA256_DIGEST_LEN);
+  ot_checksum_get_digest (&checksum, *out_csum, OSTREE_SHA256_DIGEST_LEN);
   return TRUE;
 }
 
@@ -945,6 +855,81 @@ ostree_checksum_file (GFile            *f,
   return TRUE;
 }
 
+/**
+ * ostree_checksum_file_at:
+ * @dfd: Directory file descriptor
+ * @path: Subpath
+ * @stbuf (allow-none): Optional stat buffer
+ * @objtype: Object type
+ * @flags: Flags
+ * @out_checksum (out) (transfer full): Return location for hex checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Compute the OSTree checksum for a given file. This is an fd-relative version
+ * of ostree_checksum_file() which also takes flags and fills in a caller
+ * allocated buffer.
+ *
+ * Since: 2017.13
+ */
+gboolean
+ostree_checksum_file_at (int               dfd,
+                         const char       *path,
+                         struct stat      *stbuf,
+                         OstreeObjectType  objtype,
+                         OstreeChecksumFlags flags,
+                         char            **out_checksum,
+                         GCancellable     *cancellable,
+                         GError          **error)
+{
+  g_return_val_if_fail (out_checksum != NULL, FALSE);
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  struct stat local_stbuf;
+  if (stbuf == NULL)
+    {
+      stbuf = &local_stbuf;
+      if (!glnx_fstatat (dfd, path, stbuf, AT_SYMLINK_NOFOLLOW, error))
+        return FALSE;
+    }
+
+  g_autoptr(GFileInfo) file_info = _ostree_stbuf_to_gfileinfo (stbuf);
+
+  g_autoptr(GInputStream) in = NULL;
+  if (S_ISREG (stbuf->st_mode))
+    {
+      glnx_autofd int fd = -1;
+      if (!glnx_openat_rdonly (dfd, path, FALSE, &fd, error))
+        return FALSE;
+      in = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE);
+    }
+  else if (S_ISLNK (stbuf->st_mode))
+    {
+      if (!ot_readlinkat_gfile_info (dfd, path, file_info, cancellable, error))
+        return FALSE;
+    }
+
+  const gboolean ignore_xattrs =
+    ((flags & OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS) > 0);
+
+  g_autoptr(GVariant) xattrs = NULL;
+  if (!ignore_xattrs && objtype == OSTREE_OBJECT_TYPE_FILE)
+    {
+      if (!glnx_dfd_name_get_all_xattrs (dfd, path, &xattrs, cancellable, error))
+        return FALSE;
+    }
+
+  g_autofree guchar *csum_bytes = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, in, objtype,
+                                        &csum_bytes, cancellable, error))
+    return FALSE;
+
+  *out_checksum = ostree_checksum_from_bytes (csum_bytes);
+  return TRUE;
+}
+
 typedef struct {
   GFile  *f;
   OstreeObjectType objtype;
@@ -1042,6 +1027,22 @@ ostree_checksum_file_async_finish (GFile          *f,
   return TRUE;
 }
 
+/* Common helper to compare checksums for an object, so we have a consistent
+ * error message.
+ */
+gboolean
+_ostree_compare_object_checksum (OstreeObjectType objtype,
+                                 const char      *expected,
+                                 const char      *actual,
+                                 GError         **error)
+{
+  if (!g_str_equal (expected, actual))
+    return glnx_throw (error, "Corrupted %s object; checksum expected='%s' actual='%s'",
+                       ostree_object_type_to_string (objtype),
+                       expected, actual);
+  return TRUE;
+}
+
 /**
  * ostree_create_directory_metadata:
  * @dir_info: a #GFileInfo containing directory information
@@ -1614,10 +1615,31 @@ _ostree_gfileinfo_equal (GFileInfo *a, GFileInfo *b)
   return TRUE;
 }
 
+/* Same motives as _ostree_gfileinfo_equal(), but for stat structs. */
+gboolean
+_ostree_stbuf_equal (struct stat *stbuf_a, struct stat *stbuf_b)
+{
+  /* trivial case */
+  if (stbuf_a == stbuf_b)
+    return TRUE;
+  if (stbuf_a->st_mode != stbuf_b->st_mode)
+    return FALSE;
+  if (S_ISREG (stbuf_a->st_mode) && (stbuf_a->st_size != stbuf_b->st_size))
+    return FALSE;
+  if (stbuf_a->st_uid != stbuf_b->st_uid)
+    return FALSE;
+  if (stbuf_a->st_gid != stbuf_b->st_gid)
+    return FALSE;
+  return TRUE;
+}
+
+/* Many parts of libostree only care about mode,uid,gid - this creates
+ * a new GFileInfo with those fields see.
+ */
 GFileInfo *
 _ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid)
 {
-  struct stat stbuf;
+  struct stat stbuf = { 0, };
   stbuf.st_mode = mode;
   stbuf.st_uid = uid;
   stbuf.st_gid = gid;

src/libostree/ostree-core.h

@@ -21,6 +21,7 @@
 
 #pragma once
 
+#include <sys/stat.h>
 #include <gio/gio.h>
 #include <ostree-types.h>
 
@@ -227,6 +228,23 @@ typedef enum {
  * Since: 2017.7
  */
 #define OSTREE_COMMIT_META_KEY_ENDOFLIFE "ostree.endoflife"
+/**
+ * OSTREE_COMMIT_META_KEY_SOURCE_TITLE:
+ *
+ * GVariant type `s`. This should hold a relatively short single line value
+ * containing a human-readable "source" for a commit, intended to be displayed
+ * near the origin ref.  This is particularly useful for systems that inject
+ * content into an OSTree commit from elsewhere - for example, generating from
+ * an OCI or qcow2 image. Or if generating from packages, the enabled repository
+ * names and their versions.
+ *
+ * Try to keep this key short (e.g. < 80 characters) and human-readable; if you
+ * desire machine readable data, consider injecting separate metadata keys.
+ *
+ * Since: 2017.13
+ */
+#define OSTREE_COMMIT_META_KEY_SOURCE_TITLE "ostree.source-title"
+
 /**
  * OSTREE_COMMIT_META_KEY_REF_BINDING:
  *
@@ -420,6 +438,26 @@ gboolean ostree_checksum_file (GFile             *f,
                                GCancellable      *cancellable,
                                GError           **error);
 
+/**
+ * OstreeChecksumFlags:
+ *
+ * Since: 2017.13
+ */
+typedef enum {
+  OSTREE_CHECKSUM_FLAGS_NONE = 0,
+  OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS = (1 << 0),
+} OstreeChecksumFlags;
+
+_OSTREE_PUBLIC
+gboolean ostree_checksum_file_at (int               dfd,
+                                  const char       *path,
+                                  struct stat      *stbuf,
+                                  OstreeObjectType  objtype,
+                                  OstreeChecksumFlags flags,
+                                  char            **out_checksum,
+                                  GCancellable     *cancellable,
+                                  GError          **error);
+
 _OSTREE_PUBLIC
 void ostree_checksum_file_async (GFile                 *f,
                                  OstreeObjectType       objtype,

src/libostree/ostree-fetcher-curl.c

@@ -269,13 +269,13 @@ ensure_tmpfile (FetcherRequest *req, GError **error)
 {
   if (!req->tmpf.initialized)
     {
-      if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".",
-                                          O_WRONLY | O_CLOEXEC, &req->tmpf,
-                                          error))
+      if (!_ostree_fetcher_tmpf_from_flags (req->flags, req->fetcher->tmpdir_dfd,
+                                            &req->tmpf, error))
         return FALSE;
     }
   return TRUE;
 }
+
 /* Check for completed transfers, and remove their easy handles */
 static void
 check_multi_info (OstreeFetcher *fetcher)
@@ -378,25 +378,19 @@ check_multi_info (OstreeFetcher *fetcher)
               g_autoptr(GError) local_error = NULL;
               GError **error = &local_error;
 
-              g_autofree char *tmpfile_path =
-                ostree_fetcher_generate_url_tmpname (eff_url);
               if (!ensure_tmpfile (req, error))
                 {
                   g_task_return_error (task, g_steal_pointer (&local_error));
                 }
-              /* This should match the libsoup chmod */
-              else if (fchmod (req->tmpf.fd, 0644) < 0)
+              else if (lseek (req->tmpf.fd, 0, SEEK_SET) < 0)
                 {
                   glnx_set_error_from_errno (error);
                   g_task_return_error (task, g_steal_pointer (&local_error));
                 }
-              else if (!glnx_link_tmpfile_at (&req->tmpf, GLNX_LINK_TMPFILE_REPLACE,
-                                              fetcher->tmpdir_dfd, tmpfile_path,
-                                              error))
-                g_task_return_error (task, g_steal_pointer (&local_error));
               else
                 {
-                  g_task_return_pointer (task, g_steal_pointer (&tmpfile_path), g_free);
+                  /* We return the tmpfile in the _finish wrapper */
+                  g_task_return_boolean (task, TRUE);
                 }
             }
         }
@@ -693,7 +687,7 @@ adopt_steal_mainctx (OstreeFetcher *self,
       guint64 readytime = g_source_get_ready_time (self->timer_event);
       guint64 curtime = g_source_get_time (self->timer_event);
       guint64 timeout_micros = curtime - readytime;
-      if (timeout_micros < 0)
+      if (curtime < readytime)
         timeout_micros = 0;
       update_timeout_cb (self->multi, timeout_micros / 1000, self);
     }
@@ -706,7 +700,7 @@ static void
 initiate_next_curl_request (FetcherRequest *req,
                             GTask *task)
 {
-  CURLMcode rc;
+  CURLcode rc;
   OstreeFetcher *self = req->fetcher;
 
   if (req->easy)
@@ -802,8 +796,8 @@ initiate_next_curl_request (FetcherRequest *req,
   curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);
   curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);
 
-  rc = curl_multi_add_handle (self->multi, req->easy);
-  g_assert (rc == CURLM_OK);
+  CURLMcode multi_rc = curl_multi_add_handle (self->multi, req->easy);
+  g_assert (multi_rc == CURLM_OK);
 }
 
 static void
@@ -887,26 +881,21 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
 gboolean
 _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                            GAsyncResult  *result,
-                                           char         **out_filename,
+                                           GLnxTmpfile   *out_tmpf,
                                            GError       **error)
 {
-  GTask *task;
-  FetcherRequest *req;
-  gpointer ret;
-
   g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
   g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
 
-  task = (GTask*)result;
-  req = g_task_get_task_data (task);
+  GTask *task = (GTask*)result;
+  FetcherRequest *req = g_task_get_task_data (task);
 
-  ret = g_task_propagate_pointer (task, error);
-  if (!ret)
+  if (!g_task_propagate_boolean (task, error))
     return FALSE;
 
   g_assert (!req->is_membuf);
-  g_assert (out_filename);
-  *out_filename = ret;
+  *out_tmpf = req->tmpf;
+  req->tmpf.initialized = FALSE; /* Transfer ownership */
 
   return TRUE;
 }

src/libostree/ostree-fetcher-soup.c

@@ -905,11 +905,8 @@ on_stream_read (GObject        *object,
     {
       if (!pending->is_membuf)
         {
-          if (!glnx_open_tmpfile_linkable_at (pending->thread_closure->base_tmpdir_dfd, ".",
-                                              O_WRONLY | O_CLOEXEC, &pending->tmpf, &local_error))
-            goto out;
-          /* This should match the libcurl chmod */
-          if (!glnx_fchmod (pending->tmpf.fd, 0644, &local_error))
+          if (!_ostree_fetcher_tmpf_from_flags (pending->flags, pending->thread_closure->base_tmpdir_dfd,
+                                                &pending->tmpf, &local_error))
             goto out;
           pending->out_stream = g_unix_output_stream_new (pending->tmpf.fd, FALSE);
         }
@@ -943,18 +940,13 @@ on_stream_read (GObject        *object,
         }
       else
         {
-          g_autofree char *uristring =
-            soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
-          g_autofree char *tmpfile_path =
-            ostree_fetcher_generate_url_tmpname (uristring);
-          if (!glnx_link_tmpfile_at (&pending->tmpf, GLNX_LINK_TMPFILE_REPLACE,
-                                     pending->thread_closure->base_tmpdir_dfd, tmpfile_path,
-                                     &local_error))
-            g_task_return_error (task, g_steal_pointer (&local_error));
+          if (lseek (pending->tmpf.fd, 0, SEEK_SET) < 0)
+            {
+              glnx_set_error_from_errno (&local_error);
+              g_task_return_error (task, g_steal_pointer (&local_error));
+            }
           else
-            g_task_return_pointer (task,
-                                   g_steal_pointer (&tmpfile_path),
-                                   (GDestroyNotify) g_free);
+            g_task_return_boolean (task, TRUE);
         }
       remove_pending (pending);
     }
@@ -1174,7 +1166,7 @@ _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
 gboolean
 _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                            GAsyncResult  *result,
-                                           char         **out_filename,
+                                           GLnxTmpfile   *out_tmpf,
                                            GError       **error)
 {
   GTask *task;
@@ -1192,8 +1184,8 @@ _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
     return FALSE;
 
   g_assert (!pending->is_membuf);
-  g_assert (out_filename);
-  *out_filename = ret;
+  *out_tmpf = pending->tmpf;
+  pending->tmpf.initialized = FALSE; /* Transfer ownership */
 
   return TRUE;
 }

src/libostree/ostree-fetcher-util.h

@@ -25,14 +25,23 @@
 
 G_BEGIN_DECLS
 
-/* FIXME - delete this and replace by having fetchers simply
- * return O_TMPFILE fds, not file paths.
- */
-static inline char *
-ostree_fetcher_generate_url_tmpname (const char *url)
+static inline gboolean
+_ostree_fetcher_tmpf_from_flags (OstreeFetcherRequestFlags flags,
+                                 int                       dfd,
+                                 GLnxTmpfile              *tmpf,
+                                 GError                  **error)
 {
-  return g_compute_checksum_for_string (G_CHECKSUM_SHA256,
-                                        url, strlen (url));
+  if ((flags & OSTREE_FETCHER_REQUEST_LINKABLE) > 0)
+    {
+      if (!glnx_open_tmpfile_linkable_at (dfd, ".", O_RDWR | O_CLOEXEC, tmpf, error))
+        return FALSE;
+    }
+  else if (!glnx_open_anonymous_tmpfile (O_RDWR | O_CLOEXEC, tmpf, error))
+    return FALSE;
+
+  if (!glnx_fchmod (tmpf->fd, 0644, error))
+    return FALSE;
+  return TRUE;
 }
 
 gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,

src/libostree/ostree-fetcher.h

@@ -55,7 +55,8 @@ typedef enum {
 
 typedef enum {
   OSTREE_FETCHER_REQUEST_NUL_TERMINATION = (1 << 0),
-  OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT = (1 << 1)
+  OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT = (1 << 1),
+  OSTREE_FETCHER_REQUEST_LINKABLE = (1 << 2),
 } OstreeFetcherRequestFlags;
 
 void
@@ -124,7 +125,7 @@ void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
 
 gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
                                                     GAsyncResult  *result,
-                                                    char         **out_filename,
+                                                    GLnxTmpfile   *out_tmpf,
                                                     GError       **error);
 
 void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,

src/libostree/ostree-gpg-verifier.c

@@ -177,7 +177,7 @@ _ostree_gpg_verifier_check_signature (OstreeGpgVerifier  *self,
       for (guint i = 0; i < self->key_ascii_files->len; i++)
         {
           const char *path = self->key_ascii_files->pdata[i];
-          glnx_fd_close int fd = -1;
+          glnx_autofd int fd = -1;
           g_auto(gpgme_data_t) kdata = NULL;
 
           if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error))
@@ -272,6 +272,9 @@ _ostree_gpg_verifier_add_keyring_file (OstreeGpgVerifier  *self,
 {
   g_return_if_fail (G_IS_FILE (path));
 
+  g_autofree gchar *path_str = g_file_get_path (path);
+  g_debug ("Adding GPG keyring file %s to verifier", path_str);
+
   self->keyrings = g_list_append (self->keyrings, g_object_ref (path));
 }
 
@@ -280,8 +283,11 @@ _ostree_gpg_verifier_add_keyring_file (OstreeGpgVerifier  *self,
  */
 void
 _ostree_gpg_verifier_add_keyring_data (OstreeGpgVerifier  *self,
-                                       GBytes             *keyring)
+                                       GBytes             *keyring,
+                                       const char         *data_source)
 {
+  g_debug ("Adding GPG keyring data from %s to verifier", data_source);
+
   g_ptr_array_add (self->keyring_data, g_bytes_ref (keyring));
 }
 
@@ -289,6 +295,8 @@ void
 _ostree_gpg_verifier_add_key_ascii_file (OstreeGpgVerifier *self,
                                          const char        *path)
 {
+  g_debug ("Adding GPG key ASCII file %s to verifier", path);
+
   if (!self->key_ascii_files)
     self->key_ascii_files = g_ptr_array_new_with_free_func (g_free);
   g_ptr_array_add (self->key_ascii_files, g_strdup (path));
@@ -319,6 +327,8 @@ _ostree_gpg_verifier_add_keyring_dir_at (OstreeGpgVerifier   *self,
                                     &dfd_iter, error))
     return FALSE;
 
+  g_debug ("Adding GPG keyring dir %s to verifier", path);
+
   while (TRUE)
     {
       struct dirent *dent;
@@ -345,7 +355,7 @@ _ostree_gpg_verifier_add_keyring_dir_at (OstreeGpgVerifier   *self,
       if (g_str_equal (name, "secring.gpg"))
         continue;
 
-      glnx_fd_close int fd = -1;
+      glnx_autofd int fd = -1;
       if (!glnx_openat_rdonly (dfd_iter.fd, dent->d_name, TRUE, &fd, error))
         return FALSE;
 

src/libostree/ostree-gpg-verifier.h

@@ -65,7 +65,8 @@ gboolean      _ostree_gpg_verifier_add_global_keyring_dir (OstreeGpgVerifier  *s
                                                            GError            **error);
 
 void _ostree_gpg_verifier_add_keyring_data (OstreeGpgVerifier *self,
-                                            GBytes            *data);
+                                            GBytes            *data,
+                                            const char        *data_source);
 void _ostree_gpg_verifier_add_keyring_file (OstreeGpgVerifier *self,
                                             GFile             *path);
 

src/libostree/ostree-impl-system-generator.c

@@ -164,7 +164,7 @@ _ostree_impl_system_generator (const char *ostree_cmdline,
   /* Prepare to write to the output unit dir; we use the "normal" dir
    * that overrides /usr, but not /etc.
    */
-  glnx_fd_close int normal_dir_dfd = -1;
+  glnx_autofd int normal_dir_dfd = -1;
   if (!glnx_opendirat (AT_FDCWD, normal_dir, TRUE, &normal_dir_dfd, error))
     return FALSE;
 

src/libostree/ostree-kernel-args.c

@@ -81,7 +81,7 @@ _ostree_kernel_args_new (void)
 }
 
 void
-_ostree_kernel_arg_autofree (OstreeKernelArgs *kargs)
+_ostree_kernel_args_free (OstreeKernelArgs *kargs)
 {
   if (!kargs)
     return;
@@ -90,12 +90,6 @@ _ostree_kernel_arg_autofree (OstreeKernelArgs *kargs)
   g_free (kargs);
 }
 
-void
-_ostree_kernel_args_cleanup (void *loc)
-{
-  _ostree_kernel_arg_autofree (*((OstreeKernelArgs**)loc));
-}
-
 void
 _ostree_kernel_args_replace_take (OstreeKernelArgs   *kargs,
                                   char               *arg)

src/libostree/ostree-kernel-args.h

@@ -19,15 +19,15 @@
 
 #pragma once
 
-#include <gio/gio.h>
+#include "libglnx.h"
 
 G_BEGIN_DECLS
 
 typedef struct _OstreeKernelArgs OstreeKernelArgs;
+void _ostree_kernel_args_free (OstreeKernelArgs *kargs);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeKernelArgs, _ostree_kernel_args_free);
 
 OstreeKernelArgs *_ostree_kernel_args_new (void);
-void _ostree_kernel_args_free (OstreeKernelArgs *kargs);
-void _ostree_kernel_args_cleanup (void *loc);
 void _ostree_kernel_args_replace_take (OstreeKernelArgs  *kargs,
                                        char              *key);
 void _ostree_kernel_args_replace (OstreeKernelArgs  *kargs,

src/libostree/ostree-repo-checkout.c

@@ -228,8 +228,24 @@ create_file_copy_from_input_at (OstreeRepo     *repo,
               return glnx_throw_errno_prefix (error, "symlinkat");
             case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES:
               {
-                /* Unioning?  Let's unlink and try again */
-                (void) unlinkat (destination_dfd, destination_name, 0);
+                /* For unioning, we further bifurcate a bit; for the "process whiteouts"
+                 * mode which is really "Docker/OCI", we need to match their semantics
+                 * and handle replacing a directory with a symlink.  See also equivalent
+                 * bits for regular files in checkout_file_hardlink().
+                 */
+                if (options->process_whiteouts)
+                  {
+                    if (!glnx_shutil_rm_rf_at (destination_dfd, destination_name, NULL, error))
+                      return FALSE;
+                  }
+                else
+                  {
+                    if (unlinkat (destination_dfd, destination_name, 0) < 0)
+                      {
+                        if (G_UNLIKELY (errno != ENOENT))
+                          return glnx_throw_errno_prefix (error, "unlinkat(%s)", destination_name);
+                      }
+                  }
                 if (symlinkat (target, destination_dfd, destination_name) < 0)
                   return glnx_throw_errno_prefix (error, "symlinkat");
               }
@@ -309,7 +325,17 @@ create_file_copy_from_input_at (OstreeRepo     *repo,
           /* Handled above */
           break;
         case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES:
-          replace_mode = GLNX_LINK_TMPFILE_REPLACE;
+          /* Special case OCI/Docker - see similar code in checkout_file_hardlink()
+           * and above for symlinks.
+           */
+          if (options->process_whiteouts)
+            {
+              if (!glnx_shutil_rm_rf_at (destination_dfd, destination_name, NULL, error))
+                return FALSE;
+              /* Inherit the NOREPLACE default...we deleted whatever's there */
+            }
+          else
+            replace_mode = GLNX_LINK_TMPFILE_REPLACE;
           break;
         case OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES:
           replace_mode = GLNX_LINK_TMPFILE_NOREPLACE_IGNORE_EXIST;
@@ -374,7 +400,8 @@ hardlink_add_tmp_name (OstreeRepo              *self,
 
 static gboolean
 checkout_file_hardlink (OstreeRepo                          *self,
-                        OstreeRepoCheckoutAtOptions           *options,
+                        const char                          *checksum,
+                        OstreeRepoCheckoutAtOptions         *options,
                         const char                          *loose_path,
                         int                                  destination_dfd,
                         const char                          *destination_name,
@@ -436,10 +463,28 @@ checkout_file_hardlink (OstreeRepo                          *self,
             if (!glnx_fstatat (destination_dfd, destination_name, &dest_stbuf,
                                AT_SYMLINK_NOFOLLOW, error))
               return FALSE;
-            const gboolean is_identical =
+            gboolean is_identical =
               (src_stbuf.st_dev == dest_stbuf.st_dev &&
                src_stbuf.st_ino == dest_stbuf.st_ino);
+            if (!is_identical && (_ostree_stbuf_equal (&src_stbuf, &dest_stbuf)))
+              {
+                /* As a last resort, do a checksum comparison. This is the case currently
+                 * with rpm-ostree pkg layering where we overlay from the pkgcache repo onto
+                 * a tree checked out from the system repo. Once those are united, we
+                 * shouldn't hit this anymore. https://github.com/ostreedev/ostree/pull/1258
+                 * */
+                OstreeChecksumFlags flags = 0;
+                if (self->disable_xattrs)
+                    flags |= OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS;
 
+                g_autofree char *actual_checksum = NULL;
+                if (!ostree_checksum_file_at (destination_dfd, destination_name,
+                                              &dest_stbuf, OSTREE_OBJECT_TYPE_FILE,
+                                              flags, &actual_checksum, cancellable, error))
+                  return FALSE;
+
+                is_identical = g_str_equal (checksum, actual_checksum);
+              }
             if (is_identical)
               ret_result = HARDLINK_RESULT_SKIP_EXISTED;
             else if (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES)
@@ -448,7 +493,15 @@ checkout_file_hardlink (OstreeRepo                          *self,
                 /* Make a link with a temp name */
                 if (!hardlink_add_tmp_name (self, srcfd, loose_path, tmpname, cancellable, error))
                   return FALSE;
-                /* Rename it into place */
+                /* For OCI/Docker mode, we need to handle replacing a directory with a regular
+                 * file.  See also the equivalent code for symlinks above.
+                 */
+                if (options->process_whiteouts)
+                  {
+                    if (!glnx_shutil_rm_rf_at (destination_dfd, destination_name, NULL, error))
+                      return FALSE;
+                  }
+                /* Rename it into place - for non-OCI this will overwrite files but not directories */
                 if (!glnx_renameat (self->tmp_dir_fd, tmpname, destination_dfd, destination_name, error))
                   return FALSE;
                 ret_result = HARDLINK_RESULT_LINKED;
@@ -563,6 +616,7 @@ checkout_one_file_at (OstreeRepo                        *repo,
                  the cache, which is in "bare" form */
               _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, OSTREE_REPO_MODE_BARE);
               if (!checkout_file_hardlink (current_repo,
+                                           checksum,
                                            options,
                                            loose_path_buf,
                                            destination_dfd, destination_name,
@@ -652,7 +706,7 @@ checkout_one_file_at (OstreeRepo                        *repo,
       }
       g_mutex_unlock (&repo->cache_lock);
 
-      if (!checkout_file_hardlink (repo, options, loose_path_buf,
+      if (!checkout_file_hardlink (repo, checksum, options, loose_path_buf,
                                    destination_dfd, destination_name,
                                    FALSE, &hardlink_res,
                                    cancellable, error))
@@ -782,7 +836,7 @@ checkout_tree_at_recurse (OstreeRepo                        *self,
       }
   }
 
-  glnx_fd_close int destination_dfd = -1;
+  glnx_autofd int destination_dfd = -1;
   if (!glnx_opendirat (destination_parent_fd, destination_name, TRUE,
                        &destination_dfd, error))
     return FALSE;
@@ -947,7 +1001,7 @@ checkout_tree_at (OstreeRepo                        *self,
        * exists.
        */
       int destination_dfd = destination_parent_fd;
-      glnx_fd_close int destination_dfd_owned = -1;
+      glnx_autofd int destination_dfd_owned = -1;
       if (strcmp (destination_name, ".") != 0)
         {
           if (mkdirat (destination_parent_fd, destination_name, 0700) < 0
@@ -1156,6 +1210,24 @@ ostree_repo_checkout_at (OstreeRepo                        *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_checkout_at_options_set_devino:
+ * @opts: Checkout options
+ * @cache: (transfer none) (nullable): Devino cache
+ *
+ * This function simply assigns @cache to the `devino_to_csum_cache` member of
+ * @opts; it's only useful for introspection.
+ *
+ * Note that cache does *not* have its refcount incremented - the lifetime of
+ * @cache must be equal to or greater than that of @opts.
+ */
+void
+ostree_repo_checkout_at_options_set_devino (OstreeRepoCheckoutAtOptions *opts,
+                                            OstreeRepoDevInoCache *cache)
+{
+  opts->devino_to_csum_cache = cache;
+}
+
 static guint
 devino_hash (gconstpointer a)
 {

src/libostree/ostree-repo-commit.c

@@ -38,6 +38,22 @@
 #include "ostree-checksum-input-stream.h"
 #include "ostree-varint.h"
 
+/* In most cases, we write into a staging dir for commit, but we also allow
+ * direct writes into objects/ for e.g. hardlink imports.
+ */
+static int
+commit_dest_dfd (OstreeRepo *self)
+{
+  if (self->in_transaction)
+    return self->commit_stagedir.fd;
+  else
+    return self->objects_dir_fd;
+}
+
+/* The objects/ directory has a two-character directory prefix for checksums
+ * to avoid putting lots of files in a single directory.   This technique
+ * is quite old, but Git also uses it for example.
+ */
 gboolean
 _ostree_repo_ensure_loose_objdir_at (int             dfd,
                                      const char     *loose_path,
@@ -60,6 +76,7 @@ _ostree_repo_ensure_loose_objdir_at (int             dfd,
   return TRUE;
 }
 
+/* This GVariant is the header for content objects (regfiles and symlinks) */
 static GVariant *
 create_file_metadata (guint32       uid,
                       guint32       gid,
@@ -82,6 +99,7 @@ create_file_metadata (guint32       uid,
   return ret_metadata;
 }
 
+/* bare-user repositories store file metadata as a user xattr */
 gboolean
 _ostree_write_bareuser_metadata (int fd,
                                  guint32       uid,
@@ -137,12 +155,7 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
   char tmpbuf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (tmpbuf, checksum, objtype, self->mode);
 
-  int dest_dfd;
-  if (self->in_transaction)
-    dest_dfd = self->commit_stagedir.fd;
-  else
-    dest_dfd = self->objects_dir_fd;
-
+  int dest_dfd = commit_dest_dfd (self);
   if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, tmpbuf,
                                             cancellable, error))
     return FALSE;
@@ -158,24 +171,19 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
 /* Given a dfd+path combination (may be regular file or symlink),
  * rename it into place.
  */
-gboolean
-_ostree_repo_commit_path_final (OstreeRepo        *self,
-                                const char        *checksum,
-                                OstreeObjectType   objtype,
-                                OtCleanupUnlinkat *tmp_path,
-                                GCancellable      *cancellable,
-                                GError           **error)
+static gboolean
+commit_path_final (OstreeRepo        *self,
+                   const char        *checksum,
+                   OstreeObjectType   objtype,
+                   OtCleanupUnlinkat *tmp_path,
+                   GCancellable      *cancellable,
+                   GError           **error)
 {
   /* The final renameat() */
   char tmpbuf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (tmpbuf, checksum, objtype, self->mode);
 
-  int dest_dfd;
-  if (self->in_transaction)
-    dest_dfd = self->commit_stagedir.fd;
-  else
-    dest_dfd = self->objects_dir_fd;
-
+  int dest_dfd = commit_dest_dfd (self);
   if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, tmpbuf,
                                             cancellable, error))
     return FALSE;
@@ -295,6 +303,7 @@ commit_loose_regfile_object (OstreeRepo        *self,
   return TRUE;
 }
 
+/* This is used by OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES */
 typedef struct
 {
   goffset unpacked;
@@ -395,35 +404,26 @@ add_size_index_to_metadata (OstreeRepo        *self,
   return g_variant_ref_sink (g_variant_builder_end (builder));
 }
 
-/* Combines a check for whether or not we already have the object with
- * allocating a tempfile if we don't.  Used by the static delta code.
+/* Create a tmpfile for writing a bare file.  Currently just used
+ * by the static delta code, but will likely later be extended
+ * to be used also by the dfd_iter commit path.
  */
 gboolean
 _ostree_repo_open_content_bare (OstreeRepo          *self,
                                 const char          *checksum,
                                 guint64              content_len,
                                 GLnxTmpfile         *out_tmpf,
-                                gboolean            *out_have_object,
                                 GCancellable        *cancellable,
                                 GError             **error)
 {
-  gboolean have_obj;
-  if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE, &have_obj,
-                                      cancellable, error))
-    return FALSE;
-  /* Do we already have this object? */
-  *out_have_object = have_obj;
-  if (have_obj)
-    {
-      /* Make sure the tempfile is unset */
-      out_tmpf->initialized = 0;
-      return TRUE;
-    }
-
   return glnx_open_tmpfile_linkable_at (self->tmp_dir_fd, ".", O_WRONLY|O_CLOEXEC,
                                         out_tmpf, error);
 }
 
+/* Used by static deltas, which have a separate "push" flow for
+ * regfile objects distinct from the "pull" model used by
+ * write_content_object().
+ */
 gboolean
 _ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
                                           const char          *checksum,
@@ -446,6 +446,10 @@ _ostree_repo_commit_trusted_content_bare (OstreeRepo          *self,
                                       cancellable, error);
 }
 
+/* Allocate an O_TMPFILE, write everything from @input to it, but
+ * not exceeding @length.  Used for every object in archive repos,
+ * and content objects in all bare-type repos.
+ */
 static gboolean
 create_regular_tmpfile_linkable_with_content (OstreeRepo *self,
                                               guint64 length,
@@ -498,7 +502,12 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self,
   return TRUE;
 }
 
-/* Write a content object. */
+/* The main driver for writing a content (regfile or symlink) object.
+ * There are a variety of tricky cases here; for example, bare-user
+ * repos store symlinks as regular files.  Computing checksums
+ * is optional; if @out_csum is `NULL`, we assume the caller already
+ * knows the checksum.
+ */
 static gboolean
 write_content_object (OstreeRepo         *self,
                       const char         *expected_checksum,
@@ -611,7 +620,6 @@ write_content_object (OstreeRepo         *self,
     }
   else
     {
-      g_autoptr(GVariant) file_meta = NULL;
       g_autoptr(GConverter) zlib_compressor = NULL;
       g_autoptr(GOutputStream) compressed_out_stream = NULL;
       g_autoptr(GOutputStream) temp_out = NULL;
@@ -626,11 +634,15 @@ write_content_object (OstreeRepo         *self,
         return FALSE;
       temp_out = g_unix_output_stream_new (tmpf.fd, FALSE);
 
-      file_meta = _ostree_zlib_file_header_new (file_info, xattrs);
+      g_autoptr(GBytes) file_meta_header = _ostree_zlib_file_header_new (file_info, xattrs);
+      gsize file_meta_len;
+      const guint8* file_meta_buf = g_bytes_get_data (file_meta_header, &file_meta_len);
 
-      if (!_ostree_write_variant_with_size (temp_out, file_meta, 0, NULL, NULL,
-                                            cancellable, error))
-        return FALSE;
+      { gsize bytes_written;
+        if (!g_output_stream_write_all (temp_out, file_meta_buf, file_meta_len, &bytes_written,
+                                        cancellable, error))
+          return FALSE;
+      }
 
       if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
         {
@@ -660,10 +672,12 @@ write_content_object (OstreeRepo         *self,
   else
     {
       actual_checksum = actual_checksum_owned = ot_checksum_instream_get_string (checksum_input);
-      if (expected_checksum && strcmp (actual_checksum, expected_checksum) != 0)
-        return glnx_throw (error, "Corrupted %s object %s (actual checksum is %s)",
-                           ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE),
-                           expected_checksum, actual_checksum);
+      if (expected_checksum)
+        {
+          if (!_ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE, expected_checksum, actual_checksum,
+                                                error))
+            return FALSE;
+        }
     }
 
   g_assert (actual_checksum != NULL); /* Pacify static analysis */
@@ -723,9 +737,8 @@ write_content_object (OstreeRepo         *self,
           g_assert_not_reached ();
         }
 
-      if (!_ostree_repo_commit_path_final (self, actual_checksum, OSTREE_OBJECT_TYPE_FILE,
-                                           &tmp_unlinker,
-                                           cancellable, error))
+      if (!commit_path_final (self, actual_checksum, OSTREE_OBJECT_TYPE_FILE,
+                              &tmp_unlinker, cancellable, error))
         return FALSE;
     }
   else
@@ -766,6 +779,114 @@ write_content_object (OstreeRepo         *self,
   return TRUE;
 }
 
+/* A fast path for local commits to `bare` or `bare-user-only`
+ * repos - we basically checksum the file and do a renameat()
+ * into place.
+ *
+ * This could be enhanced down the line to handle cases where we have a modified
+ * stat struct in place; e.g. for `bare` we could do the `chown`, or chmod etc.,
+ * and reset the xattrs.
+ *
+ * We could also do this for bare-user, would just involve adding the xattr (and
+ * potentially deleting other ones...not sure if we'd really want e.g. the
+ * security.selinux xattr on setuid binaries and the like to live on).
+ */
+static gboolean
+adopt_and_commit_regfile (OstreeRepo   *self,
+                          int           dfd,
+                          const char   *name,
+                          GFileInfo    *finfo,
+                          GVariant     *xattrs,
+                          char         *out_checksum_buf,
+                          GCancellable *cancellable,
+                          GError      **error)
+{
+  g_assert (G_IN_SET (self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER_ONLY));
+  g_autoptr(GBytes) header = _ostree_file_header_new (finfo, xattrs);
+
+  g_auto(OtChecksum) hasher = { 0, };
+  ot_checksum_init (&hasher);
+  ot_checksum_update_bytes (&hasher, header);
+
+  glnx_autofd int fd = -1;
+  if (!glnx_openat_rdonly (dfd, name, FALSE, &fd, error))
+    return FALSE;
+
+  (void)posix_fadvise (fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+  /* See also https://gist.github.com/cgwalters/0df0d15199009664549618c2188581f0
+   * and https://github.com/coreutils/coreutils/blob/master/src/ioblksize.h
+   * Turns out bigger block size is better; down the line we should use their
+   * same heuristics.
+   */
+  char buf[16*1024];
+  while (TRUE)
+    {
+      ssize_t bytes_read = read (fd, buf, sizeof (buf));
+      if (bytes_read < 0)
+        return glnx_throw_errno_prefix (error, "read");
+      if (bytes_read == 0)
+        break;
+
+      ot_checksum_update (&hasher, (guint8*)buf, bytes_read);
+    }
+
+  ot_checksum_get_hexdigest (&hasher, out_checksum_buf, OSTREE_SHA256_STRING_LEN+1);
+  const char *checksum = out_checksum_buf;
+
+  /* TODO: dedup this with commit_path_final() */
+  char loose_path[_OSTREE_LOOSE_PATH_MAX];
+  _ostree_loose_path (loose_path, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
+
+  const guint32 src_dev = g_file_info_get_attribute_uint32 (finfo, "unix::device");
+  const guint64 src_inode = g_file_info_get_attribute_uint64 (finfo, "unix::inode");
+
+  int dest_dfd = commit_dest_dfd (self);
+  if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, loose_path,
+                                            cancellable, error))
+    return FALSE;
+
+  struct stat dest_stbuf;
+  if (!glnx_fstatat_allow_noent (dest_dfd, loose_path, &dest_stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  /* Is the source actually the same device/inode? This can happen with hardlink
+   * checkouts, which is a bit overly conservative for bare-user-only right now.
+   * If so, we can't use renameat() since from `man 2 renameat`:
+   *
+   * "If oldpath and newpath are existing hard links referring to the same file,
+   * then rename() does nothing, and returns a success status."
+   */
+  if (errno != ENOENT
+      && src_dev == dest_stbuf.st_dev
+      && src_inode == dest_stbuf.st_ino)
+    {
+      if (!glnx_unlinkat (dfd, name, 0, error))
+        return FALSE;
+
+      /* Early return */
+      return TRUE;
+    }
+
+  /* For bare-user-only we need to canonicalize perms */
+  if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
+    {
+      const guint32 src_mode = g_file_info_get_attribute_uint32 (finfo, "unix::mode");
+      if (fchmod (fd, src_mode & 0755) < 0)
+        return glnx_throw_errno_prefix (error, "fchmod");
+    }
+  if (renameat (dfd, name, dest_dfd, loose_path) == -1)
+    {
+      if (errno != EEXIST)
+        return glnx_throw_errno_prefix (error, "Storing file '%s'", name);
+      /* We took ownership here, so delete it */
+      if (!glnx_unlinkat (dfd, name, 0, error))
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+/* Main driver for writing a metadata (non-content) object. */
 static gboolean
 write_metadata_object (OstreeRepo         *self,
                        OstreeObjectType    objtype,
@@ -790,14 +911,19 @@ write_metadata_object (OstreeRepo         *self,
    * *original* sha256 to say what commit was being killed.
    */
   const gboolean is_tombstone = (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT);
-  g_autofree char *actual_checksum = NULL;
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
   if (is_tombstone)
     {
-      actual_checksum = g_strdup (expected_checksum);
+      memcpy (actual_checksum, expected_checksum, sizeof (actual_checksum));
     }
   else
     {
-      actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, buf);
+      OtChecksum checksum = { 0, };
+      ot_checksum_init (&checksum);
+      gsize len;
+      const guint8*bufdata = g_bytes_get_data (buf, &len);
+      ot_checksum_update (&checksum, bufdata, len);
+      ot_checksum_get_hexdigest (&checksum, actual_checksum, sizeof (actual_checksum));
       gboolean have_obj;
       if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype, &have_obj,
                                           cancellable, error))
@@ -817,10 +943,11 @@ write_metadata_object (OstreeRepo         *self,
           return TRUE;
         }
 
-      if (expected_checksum && strcmp (actual_checksum, expected_checksum) != 0)
-        return glnx_throw (error, "Corrupted %s object %s (actual checksum is %s)",
-                           ostree_object_type_to_string (objtype),
-                           expected_checksum, actual_checksum);
+      if (expected_checksum)
+        {
+          if (!_ostree_compare_object_checksum (objtype, expected_checksum, actual_checksum, error))
+            return FALSE;
+        }
     }
 
   /* Ok, checksum is known, let's get the data */
@@ -888,6 +1015,9 @@ write_metadata_object (OstreeRepo         *self,
   return TRUE;
 }
 
+/* Look in a single subdirectory of objects/, building up the
+ * (device,inode) → checksum map.
+ */
 static gboolean
 scan_one_loose_devino (OstreeRepo                     *self,
                        int                             object_dir_fd,
@@ -969,6 +1099,7 @@ scan_one_loose_devino (OstreeRepo                     *self,
   return TRUE;
 }
 
+/* Used by ostree_repo_scan_hardlinks(); see that function for more information. */
 static gboolean
 scan_loose_devino (OstreeRepo                     *self,
                    GHashTable                     *devino_cache,
@@ -995,6 +1126,8 @@ scan_loose_devino (OstreeRepo                     *self,
   return TRUE;
 }
 
+/* Loook up a (device,inode) pair in our cache, and see if it maps to a known
+ * checksum. */
 static const char *
 devino_cache_lookup (OstreeRepo           *self,
                      OstreeRepoCommitModifier *modifier,
@@ -1122,6 +1255,12 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
   return TRUE;
 }
 
+/* Called for commit, to iterate over the "staging" directory and rename all the
+ * objects into the primary objects/ location. Notably this is called only after
+ * syncfs() has potentially been invoked to ensure that all objects have been
+ * written to disk.  In the future we may enhance this; see
+ * https://github.com/ostreedev/ostree/issues/1184
+ */
 static gboolean
 rename_pending_loose_objects (OstreeRepo        *self,
                               GCancellable      *cancellable,
@@ -1189,7 +1328,7 @@ rename_pending_loose_objects (OstreeRepo        *self,
           /* Ensure that in the case of a power cut all the directory metadata that
              we want has reached the disk. In particular, we want this before we
              update the refs to point to these objects. */
-          glnx_fd_close int target_dir_fd = -1;
+          glnx_autofd int target_dir_fd = -1;
 
           loose_objpath[2] = 0;
 
@@ -1217,6 +1356,12 @@ rename_pending_loose_objects (OstreeRepo        *self,
   return TRUE;
 }
 
+/* Look in repo/tmp and delete files that are older than a day (by default).
+ * This used to be primarily used by the libsoup fetcher which stored partially
+ * written objects.  In practice now that that isn't done anymore, we should
+ * use different logic here.  Some more information in
+ * https://github.com/ostreedev/ostree/issues/713
+ */
 static gboolean
 cleanup_tmpdir (OstreeRepo        *self,
                 GCancellable      *cancellable,
@@ -1560,6 +1705,17 @@ ostree_repo_commit_transaction (OstreeRepo                  *self,
   return TRUE;
 }
 
+/**
+ * ostree_repo_abort_transaction:
+ * @self: An #OstreeRepo
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Abort the active transaction; any staged objects and ref changes will be
+ * discarded. You *must* invoke this if you have chosen not to invoke
+ * ostree_repo_commit_transaction(). Calling this function when not in a
+ * transaction will do nothing and return successfully.
+ */
 gboolean
 ostree_repo_abort_transaction (OstreeRepo     *self,
                                GCancellable   *cancellable,
@@ -1814,6 +1970,15 @@ ostree_repo_write_metadata_async (OstreeRepo               *self,
   g_object_unref (asyncdata->result);
 }
 
+/**
+ * ostree_repo_write_metadata_finish:
+ * @self: Repo
+ * @result: Result
+ * @out_csum: (out) (array fixed-size=32) (element-type guint8): Binary checksum value
+ * @error: Error
+ *
+ * Complete a call to ostree_repo_write_metadata_async().
+ */
 gboolean
 ostree_repo_write_metadata_finish (OstreeRepo        *self,
                                    GAsyncResult      *result,
@@ -1835,6 +2000,9 @@ ostree_repo_write_metadata_finish (OstreeRepo        *self,
   return TRUE;
 }
 
+/* Write an object of type OSTREE_OBJECT_TYPE_DIR_META, using @file_info and @xattrs.
+ * Return its (binary) checksum in @out_csum.
+ */
 gboolean
 _ostree_repo_write_directory_meta (OstreeRepo   *self,
                                    GFileInfo    *file_info,
@@ -1843,13 +2011,11 @@ _ostree_repo_write_directory_meta (OstreeRepo   *self,
                                    GCancellable *cancellable,
                                    GError      **error)
 {
-  g_autoptr(GVariant) dirmeta = NULL;
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
 
-  dirmeta = ostree_create_directory_metadata (file_info, xattrs);
-
+  g_autoptr(GVariant) dirmeta = ostree_create_directory_metadata (file_info, xattrs);
   return ostree_repo_write_metadata (self, OSTREE_OBJECT_TYPE_DIR_META, NULL,
                                      dirmeta, out_csum, cancellable, error);
 }
@@ -2152,26 +2318,29 @@ ostree_repo_read_commit_detached_metadata (OstreeRepo      *self,
   char buf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, self->mode);
 
-  g_autoptr(GVariant) ret_metadata = NULL;
-  if (self->commit_stagedir.initialized &&
-      !ot_util_variant_map_at (self->commit_stagedir.fd, buf,
-                               G_VARIANT_TYPE ("a{sv}"),
-                               OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error))
-    return glnx_prefix_error (error, "Unable to read existing detached metadata");
+  if (self->commit_stagedir.initialized)
+    {
+      glnx_autofd int fd = -1;
+      if (!ot_openat_ignore_enoent (self->commit_stagedir.fd, buf, &fd, error))
+        return FALSE;
+      if (fd != -1)
+        return ot_variant_read_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"), TRUE,
+                                   out_metadata, error);
+    }
 
-  if (ret_metadata == NULL &&
-      !ot_util_variant_map_at (self->objects_dir_fd, buf,
-                               G_VARIANT_TYPE ("a{sv}"),
-                               OT_VARIANT_MAP_ALLOW_NOENT | OT_VARIANT_MAP_TRUSTED, &ret_metadata, error))
-    return glnx_prefix_error (error, "Unable to read existing detached metadata");
+  glnx_autofd int fd = -1;
+  if (!ot_openat_ignore_enoent (self->objects_dir_fd, buf, &fd, error))
+    return FALSE;
+  if (fd != -1)
+    return ot_variant_read_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"), TRUE,
+                               out_metadata, error);
 
-  if (ret_metadata == NULL && self->parent_repo)
+  if (self->parent_repo)
     return ostree_repo_read_commit_detached_metadata (self->parent_repo,
-                                                      checksum,
-                                                      out_metadata,
-                                                      cancellable,
-                                                      error);
-  ot_transfer_out_value (out_metadata, &ret_metadata);
+                                                      checksum, out_metadata,
+                                                      cancellable, error);
+  /* Nothing found */
+  *out_metadata = NULL;
   return TRUE;
 }
 
@@ -2230,6 +2399,9 @@ ostree_repo_write_commit_detached_metadata (OstreeRepo      *self,
   return TRUE;
 }
 
+/* This generates an in-memory OSTREE_OBJECT_TYPE_DIR_TREE variant, using the
+ * content objects and subdirectories. The input hashes will be sorted
+ */
 static GVariant *
 create_tree_variant_from_hashes (GHashTable            *file_checksums,
                                  GHashTable            *dir_contents_checksums,
@@ -2341,6 +2513,7 @@ _ostree_repo_commit_modifier_apply (OstreeRepo               *self,
   return result;
 }
 
+/* Convert @path into a string */
 static char *
 ptrarray_path_join (GPtrArray  *path)
 {
@@ -2370,6 +2543,7 @@ get_final_xattrs (OstreeRepo                       *self,
                   GFile                            *path,
                   int                               dfd,
                   const char                       *dfd_subpath,
+                  GVariant                         *source_xattrs,
                   GVariant                        **out_xattrs,
                   gboolean                         *out_modified,
                   GCancellable                     *cancellable,
@@ -2385,7 +2559,9 @@ get_final_xattrs (OstreeRepo                       *self,
   g_autoptr(GVariant) original_xattrs = NULL;
   if (!skip_xattrs && !self->disable_xattrs)
     {
-      if (path && OSTREE_IS_REPO_FILE (path))
+      if (source_xattrs)
+        original_xattrs = g_variant_ref (source_xattrs);
+      else if (path && OSTREE_IS_REPO_FILE (path))
         {
           if (!ostree_repo_file_get_xattrs (OSTREE_REPO_FILE (path), &original_xattrs,
                                             cancellable, error))
@@ -2494,6 +2670,11 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
                                   GCancellable                *cancellable,
                                   GError                     **error);
 
+typedef enum {
+  WRITE_DIR_CONTENT_FLAGS_NONE = 0,
+  WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT = 1,
+} WriteDirContentFlags;
+
 /* Given either a dir_enum or a dfd_iter, writes the directory entry to the mtree. For
  * subdirs, we go back through either write_dfd_iter_to_mtree_internal (dfd_iter case) or
  * write_directory_to_mtree_internal (dir_enum case) which will do the actual dirmeta +
@@ -2503,6 +2684,7 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
                                            OstreeRepoFile              *repo_dir,
                                            GFileEnumerator             *dir_enum,
                                            GLnxDirFdIterator           *dfd_iter,
+                                           WriteDirContentFlags         writeflags,
                                            GFileInfo                   *child_info,
                                            OstreeMutableTree           *mtree,
                                            OstreeRepoCommitModifier    *modifier,
@@ -2512,24 +2694,62 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
 {
   g_assert (dir_enum != NULL || dfd_iter != NULL);
 
+  GFileType file_type = g_file_info_get_file_type (child_info);
+
   const char *name = g_file_info_get_name (child_info);
   g_ptr_array_add (path, (char*)name);
 
   g_autofree char *child_relpath = ptrarray_path_join (path);
 
+  /* See if we have a devino hit; this is used below. Further, for bare-user
+   * repos we'll reload our file info from the object (specifically the
+   * ostreemeta xattr). The on-disk state is not normally what we want to
+   * commit. Basically we're making sure that we pick up "real" uid/gid and any
+   * xattrs there.
+   */
+  const char *loose_checksum = NULL;
+  g_autoptr(GVariant) source_xattrs = NULL;
+  if (dfd_iter != NULL && (file_type != G_FILE_TYPE_DIRECTORY))
+    {
+      guint32 dev = g_file_info_get_attribute_uint32 (child_info, "unix::device");
+      guint64 inode = g_file_info_get_attribute_uint64 (child_info, "unix::inode");
+      loose_checksum = devino_cache_lookup (self, modifier, dev, inode);
+      if (loose_checksum && self->mode == OSTREE_REPO_MODE_BARE_USER)
+        {
+          child_info = NULL;
+          if (!ostree_repo_load_file (self, loose_checksum, NULL, &child_info, &source_xattrs,
+                                      cancellable, error))
+            return FALSE;
+        }
+    }
+
   g_autoptr(GFileInfo) modified_info = NULL;
   OstreeRepoCommitFilterResult filter_result =
     _ostree_repo_commit_modifier_apply (self, modifier, child_relpath, child_info, &modified_info);
   const gboolean child_info_was_modified = !_ostree_gfileinfo_equal (child_info, modified_info);
 
+  /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even
+   * more complexity in this function, and it'd mostly only be useful when
+   * operating on local filesystems anyways.
+   */
+  const gboolean delete_after_commit = dfd_iter && modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+  const gboolean canonical_permissions = modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS);
+
   if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW)
     {
       g_ptr_array_remove_index (path, path->len - 1);
+      if (delete_after_commit)
+        {
+          g_assert (dfd_iter);
+          if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
+            return FALSE;
+        }
       /* Note: early return */
       return TRUE;
     }
 
-  GFileType file_type = g_file_info_get_file_type (child_info);
   switch (file_type)
     {
     case G_FILE_TYPE_DIRECTORY:
@@ -2568,6 +2788,12 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
                                                  modifier, path,
                                                  cancellable, error))
             return FALSE;
+
+          if (delete_after_commit)
+            {
+              if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
+                return FALSE;
+            }
         }
     }
   else if (repo_dir)
@@ -2581,67 +2807,139 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
     }
   else
     {
-      guint64 file_obj_length;
-      g_autoptr(GInputStream) file_input = NULL;
-      g_autoptr(GInputStream) file_object_input = NULL;
-      g_autofree guchar *child_file_csum = NULL;
-      g_autofree char *tmp_checksum = NULL;
+      glnx_autofd int file_input_fd = -1;
+
+      /* Open the file now, since it's better for reading xattrs
+       * rather than using the /proc/self/fd links.
+       *
+       * TODO: Do this lazily, since for e.g. bare-user-only repos
+       * we don't have xattrs and don't need to open every file
+       * for things that have devino cache hits.
+       */
+      if (file_type == G_FILE_TYPE_REGULAR && dfd_iter != NULL)
+        {
+          if (!glnx_openat_rdonly (dfd_iter->fd, name, FALSE, &file_input_fd, error))
+            return FALSE;
+        }
 
       g_autoptr(GVariant) xattrs = NULL;
       gboolean xattrs_were_modified;
-      if (!get_final_xattrs (self, modifier, child_relpath, child_info, child,
-                             dfd_iter != NULL ? dfd_iter->fd : -1, name, &xattrs,
-                             &xattrs_were_modified, cancellable, error))
-        return FALSE;
-
-      /* only check the devino cache if the file info & xattrs were not modified */
-      const char *loose_checksum = NULL;
-      if (!child_info_was_modified && !xattrs_were_modified)
+      if (dir_enum != NULL)
         {
-          guint32 dev = g_file_info_get_attribute_uint32 (child_info, "unix::device");
-          guint64 inode = g_file_info_get_attribute_uint64 (child_info, "unix::inode");
-          loose_checksum = devino_cache_lookup (self, modifier, dev, inode);
+          if (!get_final_xattrs (self, modifier, child_relpath, child_info, child,
+                                 -1, name, source_xattrs, &xattrs, &xattrs_were_modified,
+                                 cancellable, error))
+            return FALSE;
+        }
+      else
+        {
+          /* These contortions are basically so we use glnx_fd_get_all_xattrs()
+           * for regfiles, and glnx_dfd_name_get_all_xattrs() for symlinks.
+           */
+          int xattr_fd_arg = (file_input_fd != -1) ? file_input_fd : dfd_iter->fd;
+          const char *xattr_path_arg = (file_input_fd != -1) ? NULL : name;
+          if (!get_final_xattrs (self, modifier, child_relpath, child_info, child,
+                                 xattr_fd_arg, xattr_path_arg, source_xattrs,
+                                 &xattrs, &xattrs_were_modified,
+                                 cancellable, error))
+            return FALSE;
         }
 
-      if (loose_checksum)
+      /* Used below to see whether we can do a fast path commit */
+      const gboolean modified_file_meta = child_info_was_modified || xattrs_were_modified;
+
+      /* A big prerequisite list of conditions for whether or not we can
+       * "adopt", i.e. just checksum and rename() into place
+       */
+      const gboolean can_adopt_basic =
+        file_type == G_FILE_TYPE_REGULAR
+        && dfd_iter != NULL
+        && delete_after_commit
+        && ((writeflags & WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT) > 0);
+      gboolean can_adopt = can_adopt_basic;
+      /* If basic prerquisites are met, check repo mode specific ones */
+      if (can_adopt)
+        {
+          /* For bare repos, we could actually chown/reset the xattrs, but let's
+           * do the basic optimizations here first.
+           */
+          if (self->mode == OSTREE_REPO_MODE_BARE)
+            can_adopt = !modified_file_meta;
+          else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
+            can_adopt = canonical_permissions;
+          else
+            /* This covers bare-user and archive.  See comments in adopt_and_commit_regfile()
+             * for notes on adding bare-user later here.
+             */
+            can_adopt = FALSE;
+        }
+      gboolean did_adopt = FALSE;
+
+      /* The very fast path - we have a devino cache hit, nothing to write */
+      if (loose_checksum && !modified_file_meta)
         {
           if (!ostree_mutable_tree_replace_file (mtree, name, loose_checksum,
                                                  error))
             return FALSE;
         }
+      /* Next fast path - we can "adopt" the file */
+      else if (can_adopt)
+        {
+          char checksum[OSTREE_SHA256_STRING_LEN+1];
+          if (!adopt_and_commit_regfile (self, dfd_iter->fd, name, modified_info, xattrs,
+                                         checksum, cancellable, error))
+            return FALSE;
+          if (!ostree_mutable_tree_replace_file (mtree, name, checksum, error))
+            return FALSE;
+          did_adopt = TRUE;
+        }
       else
         {
-          if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR)
+          g_autoptr(GInputStream) file_input = NULL;
+
+          if (file_type == G_FILE_TYPE_REGULAR)
             {
-              if (child != NULL)
+              if (dir_enum != NULL)
                 {
+                  g_assert (child != NULL);
                   file_input = (GInputStream*)g_file_read (child, cancellable, error);
                   if (!file_input)
                     return FALSE;
                 }
               else
                 {
-                  if (!ot_openat_read_stream (dfd_iter->fd, name, FALSE,
-                                              &file_input, cancellable, error))
-                    return FALSE;
+                  /* We already opened the fd above */
+                  file_input = g_unix_input_stream_new (file_input_fd, FALSE);
                 }
             }
 
+          g_autoptr(GInputStream) file_object_input = NULL;
+          guint64 file_obj_length;
           if (!ostree_raw_file_to_content_stream (file_input,
                                                   modified_info, xattrs,
                                                   &file_object_input, &file_obj_length,
                                                   cancellable, error))
             return FALSE;
+          g_autofree guchar *child_file_csum = NULL;
           if (!ostree_repo_write_content (self, NULL, file_object_input, file_obj_length,
                                           &child_file_csum, cancellable, error))
             return FALSE;
 
-          g_free (tmp_checksum);
-          tmp_checksum = ostree_checksum_from_bytes (child_file_csum);
+          char tmp_checksum[OSTREE_SHA256_STRING_LEN+1];
+          ostree_checksum_inplace_from_bytes (child_file_csum, tmp_checksum);
           if (!ostree_mutable_tree_replace_file (mtree, name, tmp_checksum,
                                                  error))
             return FALSE;
         }
+
+      /* Process delete_after_commit. In the adoption case though, we already
+       * took ownership of the file above, usually via a renameat().
+       */
+      if (delete_after_commit && !did_adopt)
+        {
+          if (!glnx_unlinkat (dfd_iter->fd, name, 0, error))
+            return FALSE;
+        }
     }
 
   g_ptr_array_remove_index (path, path->len - 1);
@@ -2701,7 +2999,7 @@ write_directory_to_mtree_internal (OstreeRepo                  *self,
       if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
         {
           if (!get_final_xattrs (self, modifier, relpath, child_info, dir, -1, NULL,
-                                 &xattrs, NULL, cancellable, error))
+                                 NULL, &xattrs, NULL, cancellable, error))
             return FALSE;
 
           g_autofree guchar *child_file_csum = NULL;
@@ -2736,6 +3034,7 @@ write_directory_to_mtree_internal (OstreeRepo                  *self,
             break;
 
           if (!write_directory_content_to_mtree_internal (self, repo_dir, dir_enum, NULL,
+                                                          WRITE_DIR_CONTENT_FLAGS_NONE,
                                                           child_info,
                                                           mtree, modifier, path,
                                                           cancellable, error))
@@ -2786,7 +3085,7 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
   if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
     {
       if (!get_final_xattrs (self, modifier, relpath, modified_info, NULL, src_dfd_iter->fd,
-                             NULL, &xattrs, NULL, cancellable, error))
+                             NULL, NULL, &xattrs, NULL, cancellable, error))
         return FALSE;
 
       if (!_ostree_repo_write_directory_meta (self, modified_info, xattrs, &child_file_csum,
@@ -2804,6 +3103,11 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
       return TRUE;
     }
 
+  /* See if this dir is on the same device; if so we can adopt (if enabled) */
+  WriteDirContentFlags flags = 0;
+  if (dir_stbuf.st_dev == self->device)
+    flags |= WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT;
+
   while (TRUE)
     {
       struct dirent *dent;
@@ -2836,7 +3140,7 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
         }
 
       if (!write_directory_content_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
-                                                      child_info,
+                                                      flags, child_info,
                                                       mtree, modifier, path,
                                                       cancellable, error))
         return FALSE;
@@ -2922,6 +3226,19 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo                *self,
                                          cancellable, error))
     return FALSE;
 
+  /* And now finally remove the toplevel; see also the handling for this flag in
+   * the write_dfd_iter_to_mtree_internal() function. As a special case we don't
+   * try to remove `.` (since we'd get EINVAL); that's what's used in
+   * rpm-ostree.
+   */
+  const gboolean delete_after_commit = modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+  if (delete_after_commit && !g_str_equal (path, "."))
+    {
+      if (!glnx_unlinkat (dfd, path, AT_REMOVEDIR, error))
+        return FALSE;
+    }
+
   return TRUE;
 }
 
@@ -3118,6 +3435,8 @@ ostree_repo_commit_modifier_set_sepolicy (OstreeRepoCommitModifier
  *
  * This function will add a reference to @cache without copying - you
  * should avoid further mutation of the cache.
+ *
+ * Since: 2017.13
  */
 void
 ostree_repo_commit_modifier_set_devino_cache (OstreeRepoCommitModifier              *modifier,
@@ -3255,7 +3574,7 @@ import_one_object_direct (OstreeRepo    *dest_repo,
       if (linkat (src_repo->objects_dir_fd, loose_path_buf, dest_dfd, loose_path_buf, 0) != 0)
         {
           if (errno == EEXIST)
-            return TRUE;
+            did_hardlink = TRUE;
           else if (errno == EMLINK || errno == EXDEV || errno == EPERM)
             {
               /* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do
@@ -3292,7 +3611,7 @@ import_one_object_direct (OstreeRepo    *dest_repo,
        * that basically just optionally does chown().  Perhaps
        * in the future we should add flags for those things?
        */
-      glnx_fd_close int src_fd = -1;
+      glnx_autofd int src_fd = -1;
       if (!glnx_openat_rdonly (src_repo->objects_dir_fd, loose_path_buf,
                                FALSE, &src_fd, error))
         return FALSE;

src/libostree/ostree-repo-finder-avahi.c

@@ -429,7 +429,7 @@ fill_refs_and_checksums_from_summary_map (GVariantIter  *summary_map,
   g_autofree gchar *ref_name = NULL;
   g_autoptr(GVariant) checksum_variant = NULL;
 
-  while (g_variant_iter_next (summary_map, "(s(t@aya{sv}))",
+  while (g_variant_iter_loop (summary_map, "(s(t@aya{sv}))",
                               (gpointer *) &ref_name, NULL,
                               (gpointer *) &checksum_variant, NULL))
     {
@@ -466,7 +466,7 @@ fill_refs_and_checksums_from_summary (GVariant    *summary,
 {
   g_autoptr(GVariant) ref_map_v = NULL;
   g_autoptr(GVariant) additional_metadata_v = NULL;
-  GVariantIter ref_map;
+  g_autoptr(GVariantIter) ref_map = NULL;
   g_auto(GVariantDict) additional_metadata = OT_VARIANT_BUILDER_INITIALIZER;
   const gchar *collection_id;
   g_autoptr(GVariantIter) collection_map = NULL;
@@ -474,7 +474,7 @@ fill_refs_and_checksums_from_summary (GVariant    *summary,
   ref_map_v = g_variant_get_child_value (summary, 0);
   additional_metadata_v = g_variant_get_child_value (summary, 1);
 
-  g_variant_iter_init (&ref_map, ref_map_v);
+  ref_map = g_variant_iter_new (ref_map_v);
   g_variant_dict_init (&additional_metadata, additional_metadata_v);
 
   /* If the summary file specifies a collection ID (to apply to all the refs in its
@@ -485,10 +485,12 @@ fill_refs_and_checksums_from_summary (GVariant    *summary,
     {
       if (!ostree_validate_collection_id (collection_id, error))
         return FALSE;
-      if (!fill_refs_and_checksums_from_summary_map (&ref_map, collection_id, refs_and_checksums, error))
+      if (!fill_refs_and_checksums_from_summary_map (ref_map, collection_id, refs_and_checksums, error))
         return FALSE;
     }
 
+  g_clear_pointer (&ref_map, (GDestroyNotify) g_variant_iter_free);
+
   /* Repeat for the other collections listed in the summary. */
   if (g_variant_dict_lookup (&additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, "a{sa(s(taya{sv}))}", &collection_map))
     {
@@ -496,7 +498,7 @@ fill_refs_and_checksums_from_summary (GVariant    *summary,
         {
           if (!ostree_validate_collection_id (collection_id, error))
             return FALSE;
-          if (!fill_refs_and_checksums_from_summary_map (&ref_map, collection_id, refs_and_checksums, error))
+          if (!fill_refs_and_checksums_from_summary_map (ref_map, collection_id, refs_and_checksums, error))
             return FALSE;
         }
     }

src/libostree/ostree-repo-finder-mount.c

@@ -23,6 +23,7 @@
 #include "config.h"
 
 #include <gio/gio.h>
+#include <gio/gunixmounts.h>
 #include <glib.h>
 #include <glib-object.h>
 #include <libglnx.h>
@@ -235,8 +236,7 @@ scan_repo (int                 dfd,
     {
       g_debug ("Ignoring repository ‘%s’ on mount ‘%s’ as it’s on a different file system from the mount",
                path, mount_name);
-      g_propagate_error (error, g_steal_pointer (&local_error));
-      return FALSE;
+      return glnx_throw (error, "Repository is on a different file system from the mount");
     }
 
   /* Exclude repositories which resolve to @parent_repo. */
@@ -245,8 +245,7 @@ scan_repo (int                 dfd,
     {
       g_debug ("Ignoring repository ‘%s’ on mount ‘%s’ as it is the same as the one we are resolving",
                path, mount_name);
-      g_propagate_error (error, g_steal_pointer (&local_error));
-      return FALSE;
+      return glnx_throw (error, "Repository is the same as the one we are resolving");
     }
 
   /* List the repo’s refs and return them. */
@@ -328,9 +327,9 @@ ostree_repo_finder_mount_resolve_async (OstreeRepoFinder                  *finde
       g_autofree gchar *mount_name = NULL;
       g_autoptr(GFile) mount_root = NULL;
       g_autofree gchar *mount_root_path = NULL;
-      glnx_fd_close int mount_root_dfd = -1;
+      glnx_autofd int mount_root_dfd = -1;
       struct stat mount_root_stbuf;
-      glnx_fd_close int repos_dfd = -1;
+      glnx_autofd int repos_dfd = -1;
       gsize i;
       g_autoptr(GHashTable) repo_to_refs = NULL;  /* (element-type UriAndKeyring GHashTable) */
       GHashTable *supported_ref_to_checksum;  /* (element-type OstreeCollectionRef utf8) */
@@ -357,6 +356,23 @@ ostree_repo_finder_mount_resolve_async (OstreeRepoFinder                  *finde
           continue;
         }
 
+#if GLIB_CHECK_VERSION(2, 55, 0)
+G_GNUC_BEGIN_IGNORE_DEPRECATIONS  /* remove once GLIB_VERSION_MAX_ALLOWED ≥ 2.56 */
+      g_autoptr(GUnixMountEntry) mount_entry = g_unix_mount_at (mount_root_path, NULL);
+
+      if (mount_entry != NULL &&
+          (g_unix_is_system_fs_type (g_unix_mount_get_fs_type (mount_entry)) ||
+           g_unix_is_system_device_path (g_unix_mount_get_device_path (mount_entry))))
+        {
+          g_debug ("Ignoring mount ‘%s’ as its file system type (%s) or device "
+                   "path (%s) indicate it’s a system mount.",
+                   mount_name, g_unix_mount_get_fs_type (mount_entry),
+                   g_unix_mount_get_device_path (mount_entry));
+          continue;
+        }
+G_GNUC_END_IGNORE_DEPRECATIONS
+#endif  /* GLib 2.56.0 */
+
       /* stat() the mount root so we can later check whether the resolved
        * repositories for individual refs are on the same device (to avoid the
        * symlinks for them pointing outside the mount root). */

src/libostree/ostree-repo-finder-override.c

@@ -0,0 +1,322 @@
+/*
+ * Copyright © 2017 Endless Mobile, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Authors:
+ *  - Philip Withnall <withnall@endlessm.com>
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <gio/gio.h>
+#include <glib.h>
+#include <glib-object.h>
+#include <libglnx.h>
+
+#include "ostree-autocleanups.h"
+#include "ostree-remote-private.h"
+#include "ostree-repo.h"
+#include "ostree-repo-private.h"
+#include "ostree-repo-finder.h"
+#include "ostree-repo-finder-override.h"
+
+/**
+ * SECTION:ostree-repo-finder-override
+ * @title: OstreeRepoFinderOverride
+ * @short_description: Finds remote repositories from a list of repository URIs
+ * @stability: Unstable
+ * @include: libostree/ostree-repo-finder-override.h
+ *
+ * #OstreeRepoFinderOverride is an implementation of #OstreeRepoFinder which
+ * looks refs up in a list of remotes given by their URI, and returns the URIs
+ * which contain the refs. Duplicate remote URIs are combined into a single
+ * #OstreeRepoFinderResult which lists multiple refs.
+ *
+ * Each result is given an #OstreeRepoFinderResult.priority value of 20, which
+ * ranks its results above those from the other default #OstreeRepoFinder
+ * implementations.
+ *
+ * Results can only be returned for a ref if a remote and keyring are configured
+ * locally for the collection ID of that ref, otherwise there would be no keys
+ * available to verify signatures on commits for that ref.
+ *
+ * This is intended to be used for user-provided overrides and testing software
+ * which uses #OstreeRepoFinder. For production use, #OstreeRepoFinderConfig is
+ * recommended instead.
+ *
+ * Since: 2017.13
+ */
+
+static void ostree_repo_finder_override_iface_init (OstreeRepoFinderInterface *iface);
+
+struct _OstreeRepoFinderOverride
+{
+  GObject parent_instance;
+
+  GPtrArray *override_uris;  /* (owned) (element-type utf8) */
+};
+
+G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderOverride, ostree_repo_finder_override, G_TYPE_OBJECT,
+                         G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_override_iface_init))
+
+static gint
+results_compare_cb (gconstpointer a,
+                    gconstpointer b)
+{
+  const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a);
+  const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b);
+
+  return ostree_repo_finder_result_compare (result_a, result_b);
+}
+
+/* This must return a valid remote name (suitable for use in a refspec). */
+static gchar *
+uri_and_keyring_to_name (const gchar *uri,
+                         const gchar *keyring)
+{
+  g_autofree gchar *escaped_uri = g_uri_escape_string (uri, NULL, FALSE);
+  g_autofree gchar *escaped_keyring = g_uri_escape_string (keyring, NULL, FALSE);
+
+  /* FIXME: Need a better separator than `_`, since it’s not escaped in the input. */
+  g_autofree gchar *out = g_strdup_printf ("%s_%s", escaped_uri, escaped_keyring);
+
+  for (gsize i = 0; out[i] != '\0'; i++)
+    {
+      if (out[i] == '%')
+        out[i] = '_';
+    }
+
+  g_return_val_if_fail (ostree_validate_remote_name (out, NULL), NULL);
+
+  return g_steal_pointer (&out);
+}
+
+/* Version of ostree_repo_remote_list_collection_refs() which takes an
+ * #OstreeRemote. */
+static gboolean
+repo_remote_list_collection_refs (OstreeRepo    *repo,
+                                  const gchar   *remote_uri,
+                                  GHashTable   **out_all_refs,
+                                  GCancellable  *cancellable,
+                                  GError       **error)
+{
+  g_autofree gchar *name = uri_and_keyring_to_name (remote_uri, "");
+  g_autoptr(OstreeRemote) remote = ostree_remote_new (name);
+  g_key_file_set_string (remote->options, remote->group, "url", remote_uri);
+
+  gboolean remote_already_existed = _ostree_repo_add_remote (repo, remote);
+  gboolean success = ostree_repo_remote_list_collection_refs (repo,
+                                                              remote->name,
+                                                              out_all_refs,
+                                                              cancellable,
+                                                              error);
+
+  if (!remote_already_existed)
+    _ostree_repo_remove_remote (repo, remote);
+
+  return success;
+}
+
+static void
+ostree_repo_finder_override_resolve_async (OstreeRepoFinder                  *finder,
+                                           const OstreeCollectionRef * const *refs,
+                                           OstreeRepo                        *parent_repo,
+                                           GCancellable                      *cancellable,
+                                           GAsyncReadyCallback                callback,
+                                           gpointer                           user_data)
+{
+  OstreeRepoFinderOverride *self = OSTREE_REPO_FINDER_OVERRIDE (finder);
+  g_autoptr(GTask) task = NULL;
+  g_autoptr(GPtrArray) results = NULL;
+  const gint priority = 20;  /* arbitrarily chosen; higher priority than the others */
+  gsize i, j;
+  g_autoptr(GHashTable) repo_remote_to_refs = NULL;  /* (element-type OstreeRemote GHashTable) */
+  GHashTable *supported_ref_to_checksum;  /* (element-type OstreeCollectionRef utf8) */
+  GHashTableIter iter;
+  const gchar *remote_uri;
+  OstreeRemote *remote;
+
+  task = g_task_new (finder, cancellable, callback, user_data);
+  g_task_set_source_tag (task, ostree_repo_finder_override_resolve_async);
+  results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free);
+  repo_remote_to_refs = g_hash_table_new_full (g_direct_hash, g_direct_equal,
+                                               (GDestroyNotify) ostree_remote_unref,
+                                               (GDestroyNotify) g_hash_table_unref);
+
+  g_debug ("%s: Checking %u overrides", G_STRFUNC, self->override_uris->len);
+
+  for (i = 0; i < self->override_uris->len; i++)
+    {
+      g_autoptr(GError) local_error = NULL;
+      g_autoptr(GHashTable) remote_refs = NULL;  /* (element-type OstreeCollectionRef utf8) */
+      const gchar *checksum;
+      gboolean resolved_a_ref = FALSE;
+
+      remote_uri = self->override_uris->pdata[i];
+
+      if (!repo_remote_list_collection_refs (parent_repo, remote_uri,
+                                             &remote_refs, cancellable,
+                                             &local_error))
+        {
+          g_debug ("Ignoring remote ‘%s’ due to error loading its refs: %s",
+                   remote_uri, local_error->message);
+          g_clear_error (&local_error);
+          continue;
+        }
+
+      for (j = 0; refs[j] != NULL; j++)
+        {
+          g_autoptr(OstreeRemote) keyring_remote = NULL;
+
+          /* Look up the GPG keyring for this ref. */
+          keyring_remote = ostree_repo_resolve_keyring_for_collection (parent_repo,
+                                                                       refs[j]->collection_id,
+                                                                       cancellable, &local_error);
+
+          if (keyring_remote == NULL)
+            {
+              g_debug ("Ignoring ref (%s, %s) due to missing keyring: %s",
+                       refs[j]->collection_id, refs[j]->ref_name, local_error->message);
+              g_clear_error (&local_error);
+              continue;
+            }
+
+          if (g_hash_table_lookup_extended (remote_refs, refs[j], NULL, (gpointer *) &checksum))
+            {
+              g_autoptr(OstreeRemote) remote = NULL;
+
+              /* The requested ref is listed in the refs for this remote. Add
+               * the remote to the results, and the ref to its
+               * @supported_ref_to_checksum. */
+              g_debug ("Resolved ref (%s, %s) to remote ‘%s’.",
+                       refs[j]->collection_id, refs[j]->ref_name, remote_uri);
+              resolved_a_ref = TRUE;
+
+              /* Build an #OstreeRemote. Use the escaped URI, since remote->name
+               * is used in file paths, so needs to not contain special characters. */
+              g_autofree gchar *name = uri_and_keyring_to_name (remote_uri, keyring_remote->name);
+              remote = ostree_remote_new_dynamic (name, keyring_remote->name);
+
+              /* gpg-verify-summary is false since we use the unsigned summary file support. */
+              g_key_file_set_string (remote->options, remote->group, "url", remote_uri);
+              g_key_file_set_boolean (remote->options, remote->group, "gpg-verify", TRUE);
+              g_key_file_set_boolean (remote->options, remote->group, "gpg-verify-summary", FALSE);
+
+              supported_ref_to_checksum = g_hash_table_lookup (repo_remote_to_refs, remote);
+
+              if (supported_ref_to_checksum == NULL)
+                {
+                  supported_ref_to_checksum = g_hash_table_new_full (ostree_collection_ref_hash,
+                                                                     ostree_collection_ref_equal,
+                                                                     NULL, g_free);
+                  g_hash_table_insert (repo_remote_to_refs, ostree_remote_ref (remote), supported_ref_to_checksum  /* transfer */);
+                }
+
+              g_hash_table_insert (supported_ref_to_checksum,
+                                   (gpointer) refs[j], g_strdup (checksum));
+            }
+        }
+
+      if (!resolved_a_ref)
+        g_debug ("Ignoring remote ‘%s’ due to it not advertising any of the requested refs.",
+                 remote_uri);
+    }
+
+  /* Aggregate the results. */
+  g_hash_table_iter_init (&iter, repo_remote_to_refs);
+
+  while (g_hash_table_iter_next (&iter, (gpointer *) &remote, (gpointer *) &supported_ref_to_checksum))
+    g_ptr_array_add (results, ostree_repo_finder_result_new (remote, finder, priority, supported_ref_to_checksum, 0));
+
+  g_ptr_array_sort (results, results_compare_cb);
+
+  g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref);
+}
+
+static GPtrArray *
+ostree_repo_finder_override_resolve_finish (OstreeRepoFinder  *finder,
+                                            GAsyncResult      *result,
+                                            GError           **error)
+{
+  g_return_val_if_fail (g_task_is_valid (result, finder), NULL);
+  return g_task_propagate_pointer (G_TASK (result), error);
+}
+
+static void
+ostree_repo_finder_override_init (OstreeRepoFinderOverride *self)
+{
+  self->override_uris = g_ptr_array_new_with_free_func ((GDestroyNotify) g_free);
+}
+
+static void
+ostree_repo_finder_override_finalize (GObject *object)
+{
+  OstreeRepoFinderOverride *self = OSTREE_REPO_FINDER_OVERRIDE (object);
+
+  g_clear_pointer (&self->override_uris, g_ptr_array_unref);
+
+  G_OBJECT_CLASS (ostree_repo_finder_override_parent_class)->finalize (object);
+}
+
+static void
+ostree_repo_finder_override_class_init (OstreeRepoFinderOverrideClass *klass)
+{
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->finalize = ostree_repo_finder_override_finalize;
+}
+
+static void
+ostree_repo_finder_override_iface_init (OstreeRepoFinderInterface *iface)
+{
+  iface->resolve_async = ostree_repo_finder_override_resolve_async;
+  iface->resolve_finish = ostree_repo_finder_override_resolve_finish;
+}
+
+/**
+ * ostree_repo_finder_override_new:
+ *
+ * Create a new #OstreeRepoFinderOverride.
+ *
+ * Returns: (transfer full): a new #OstreeRepoFinderOverride
+ * Since: 2017.13
+ */
+OstreeRepoFinderOverride *
+ostree_repo_finder_override_new (void)
+{
+  return g_object_new (OSTREE_TYPE_REPO_FINDER_OVERRIDE, NULL);
+}
+
+/**
+ * ostree_repo_finder_override_add_uri:
+ * @uri: URI to add to the repo finder
+ *
+ * Add the given @uri to the set of URIs which the repo finder will search for
+ * matching refs when ostree_repo_finder_resolve_async() is called on it.
+ *
+ * Since: 2017.13
+ */
+void
+ostree_repo_finder_override_add_uri (OstreeRepoFinderOverride *self,
+                                     const gchar              *uri)
+{
+  g_return_if_fail (OSTREE_IS_REPO_FINDER_OVERRIDE (self));
+  g_return_if_fail (uri != NULL);
+
+  g_ptr_array_add (self->override_uris, g_strdup (uri));
+}

src/libostree/ostree-repo-finder-override.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright © 2017 Endless Mobile, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Authors:
+ *  - Philip Withnall <withnall@endlessm.com>
+ */
+
+#pragma once
+
+#include <gio/gio.h>
+#include <glib.h>
+#include <glib-object.h>
+
+#include "ostree-repo-finder.h"
+#include "ostree-types.h"
+
+G_BEGIN_DECLS
+
+#define OSTREE_TYPE_REPO_FINDER_OVERRIDE (ostree_repo_finder_override_get_type ())
+
+/* Manually expanded version of the following, omitting autoptr support (for GLib < 2.44):
+_OSTREE_PUBLIC
+G_DECLARE_FINAL_TYPE (OstreeRepoFinderOverride, ostree_repo_finder_override, OSTREE, REPO_FINDER_OVERRIDE, GObject) */
+
+_OSTREE_PUBLIC
+GType ostree_repo_finder_override_get_type (void);
+
+G_GNUC_BEGIN_IGNORE_DEPRECATIONS
+typedef struct _OstreeRepoFinderOverride OstreeRepoFinderOverride;
+typedef struct { GObjectClass parent_class; } OstreeRepoFinderOverrideClass;
+
+static inline OstreeRepoFinderOverride *OSTREE_REPO_FINDER_OVERRIDE (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_repo_finder_override_get_type (), OstreeRepoFinderOverride); }
+static inline gboolean OSTREE_IS_REPO_FINDER_OVERRIDE (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_repo_finder_override_get_type ()); }
+G_GNUC_END_IGNORE_DEPRECATIONS
+
+_OSTREE_PUBLIC
+OstreeRepoFinderOverride *ostree_repo_finder_override_new (void);
+
+_OSTREE_PUBLIC
+void ostree_repo_finder_override_add_uri (OstreeRepoFinderOverride *self,
+                                          const gchar              *uri);
+
+G_END_DECLS

src/libostree/ostree-repo-private.h

@@ -365,20 +365,11 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
                                 GCancellable      *cancellable,
                                 GError           **error);
 
-gboolean
-_ostree_repo_commit_path_final (OstreeRepo        *self,
-                                const char        *checksum,
-                                OstreeObjectType   objtype,
-                                OtCleanupUnlinkat *tmp_path,
-                                GCancellable      *cancellable,
-                                GError           **error);
-
 gboolean
 _ostree_repo_open_content_bare (OstreeRepo          *self,
                                 const char          *checksum,
                                 guint64              content_len,
                                 GLnxTmpfile         *out_tmpf,
-                                gboolean            *out_have_object,
                                 GCancellable        *cancellable,
                                 GError             **error);
 

src/libostree/ostree-repo-pull.c

@@ -105,11 +105,12 @@ typedef struct {
   GVariant         *summary;
   GHashTable       *summary_deltas_checksums;
   GHashTable       *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
+  GHashTable       *gpg_verified_commits; /* Set<checksum> of commits that have been verified */
   GPtrArray        *static_delta_superblocks;
   GHashTable       *expected_commit_sizes; /* Maps commit checksum to known size */
   GHashTable       *commit_to_depth; /* Maps commit checksum maximum depth */
   GHashTable       *scanned_metadata; /* Maps object name to itself */
-  GHashTable       *fetched_detached_metadata; /* Set<checksum> */
+  GHashTable       *fetched_detached_metadata; /* Map<checksum,GVariant> */
   GHashTable       *requested_metadata; /* Maps object name to itself */
   GHashTable       *requested_content; /* Maps checksum to itself */
   GHashTable       *requested_fallback_content; /* Maps checksum to itself */
@@ -142,6 +143,7 @@ typedef struct {
   guint64           start_time;
 
   gboolean          is_mirror;
+  gboolean          trusted_http_direct;
   gboolean          is_commit_only;
   OstreeRepoImportFlags importflags;
 
@@ -191,6 +193,13 @@ typedef struct {
   OstreeCollectionRef *requested_ref;  /* (nullable) */
 } ScanObjectQueueData;
 
+static void
+variant_or_null_unref (gpointer data)
+{
+  if (data)
+    g_variant_unref (data);
+}
+
 static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch);
 static void start_fetch_deltapart (OtPullData *pull_data,
                                    FetchStaticDeltaData *fetch);
@@ -209,15 +218,23 @@ static void queue_scan_one_metadata_object_c (OtPullData                *pull_da
                                               guint                      recursion_depth,
                                               const OstreeCollectionRef *ref);
 
-static gboolean scan_one_metadata_object_c (OtPullData                 *pull_data,
-                                            const guchar               *csum,
-                                            OstreeObjectType            objtype,
-                                            const char                 *path,
-                                            guint                       recursion_depth,
-                                            const OstreeCollectionRef  *ref,
-                                            GCancellable               *cancellable,
-                                            GError                    **error);
+static gboolean scan_one_metadata_object (OtPullData                 *pull_data,
+                                          const char                 *checksum,
+                                          OstreeObjectType            objtype,
+                                          const char                 *path,
+                                          guint                       recursion_depth,
+                                          const OstreeCollectionRef  *ref,
+                                          GCancellable               *cancellable,
+                                          GError                    **error);
 static void scan_object_queue_data_free (ScanObjectQueueData *scan_data);
+static gboolean
+gpg_verify_unwritten_commit (OtPullData         *pull_data,
+                             const char         *checksum,
+                             GVariant           *commit,
+                             GVariant           *detached_metadata,
+                             GCancellable       *cancellable,
+                             GError            **error);
+
 
 static gboolean
 update_progress (gpointer user_data)
@@ -450,6 +467,11 @@ scan_object_queue_data_free (ScanObjectQueueData *scan_data)
   g_free (scan_data);
 }
 
+/* Called out of the main loop to process the "scan object queue", which is a
+ * queue of metadata objects (commits and dirtree, but not dirmeta) to parse to
+ * look for further objects. Basically wraps execution of
+ * `scan_one_metadata_object()`.
+ */
 static gboolean
 idle_worker (gpointer user_data)
 {
@@ -464,14 +486,11 @@ idle_worker (gpointer user_data)
       return G_SOURCE_REMOVE;
     }
 
-  scan_one_metadata_object_c (pull_data,
-                              scan_data->csum,
-                              scan_data->objtype,
-                              scan_data->path,
-                              scan_data->recursion_depth,
-                              scan_data->requested_ref,
-                              pull_data->cancellable,
-                              &error);
+  char checksum[OSTREE_SHA256_STRING_LEN+1];
+  ostree_checksum_inplace_from_bytes (scan_data->csum, checksum);
+  scan_one_metadata_object (pull_data, checksum, scan_data->objtype,
+                            scan_data->path, scan_data->recursion_depth,
+                            scan_data->requested_ref, pull_data->cancellable, &error);
   check_outstanding_requests_handle_error (pull_data, &error);
   scan_object_queue_data_free (scan_data);
 
@@ -544,7 +563,7 @@ write_commitpartial_for (OtPullData *pull_data,
                          GError **error)
 {
   g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
-  glnx_fd_close int fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644);
+  glnx_autofd int fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644);
   if (fd == -1)
     {
       if (errno != EEXIST)
@@ -844,7 +863,7 @@ fetch_ref_contents (OtPullData                 *pull_data,
     {
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
       if (!ostree_repo_resolve_collection_ref (pull_data->remote_repo_local,
-                                               ref, TRUE  /* ignore enoent */,
+                                               ref, FALSE,
                                                OSTREE_REPO_RESOLVE_REV_EXT_NONE,
                                                &ret_contents, cancellable, error))
         return FALSE;
@@ -855,7 +874,7 @@ fetch_ref_contents (OtPullData                 *pull_data,
   else if (pull_data->remote_repo_local != NULL)
     {
       if (!ostree_repo_resolve_rev_ext (pull_data->remote_repo_local,
-                                        ref->ref_name, TRUE  /* ignore enoent */,
+                                        ref->ref_name, FALSE,
                                         OSTREE_REPO_RESOLVE_REV_EXT_NONE,
                                         &ret_contents, error))
         return FALSE;
@@ -874,14 +893,13 @@ fetch_ref_contents (OtPullData                 *pull_data,
                                                   filename, &ret_contents,
                                                   cancellable, error))
         return FALSE;
+
+      g_strchomp (ret_contents);
     }
 
-  /* Validate and return. */
-  if (ret_contents != NULL)
-    g_strchomp (ret_contents);
+  g_assert (ret_contents);
 
-  if (ret_contents == NULL ||
-      !ostree_validate_checksum_string (ret_contents, error))
+  if (!ostree_validate_checksum_string (ret_contents, error))
     return glnx_prefix_error (error, "Fetching checksum for ref (%s, %s)",
                               ref->collection_id ? ref->collection_id : "(empty)",
                               ref->ref_name);
@@ -987,13 +1005,8 @@ content_fetch_on_write_complete (GObject        *object,
   checksum_obj = ostree_object_to_string (checksum, objtype);
   g_debug ("write of %s complete", checksum_obj);
 
-  if (strcmp (checksum, expected_checksum) != 0)
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Corrupted content object; checksum expected='%s' actual='%s'",
-                   expected_checksum, checksum);
-      goto out;
-    }
+  if (!_ostree_compare_object_checksum (objtype, expected_checksum, checksum, error))
+    goto out;
 
   pull_data->n_fetched_content++;
   /* Was this a delta fallback? */
@@ -1017,17 +1030,18 @@ content_fetch_on_complete (GObject        *object,
   GError **error = &local_error;
   GCancellable *cancellable = NULL;
   guint64 length;
+  g_auto(GLnxTmpfile) tmpf = { 0, };
+  g_autoptr(GInputStream) tmpf_input = NULL;
   g_autoptr(GFileInfo) file_info = NULL;
   g_autoptr(GVariant) xattrs = NULL;
   g_autoptr(GInputStream) file_in = NULL;
   g_autoptr(GInputStream) object_input = NULL;
-  g_auto(OtCleanupUnlinkat) tmp_unlinker = { _ostree_fetcher_get_dfd (fetcher), NULL };
   const char *checksum;
   g_autofree char *checksum_obj = NULL;
   OstreeObjectType objtype;
   gboolean free_fetch_data = TRUE;
 
-  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmp_unlinker.path, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
     goto out;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -1039,48 +1053,31 @@ content_fetch_on_complete (GObject        *object,
   const gboolean verifying_bareuseronly =
     (pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY) > 0;
 
-  /* If we're mirroring and writing into an archive repo, and both checksum and
-   * bareuseronly are turned off, we can directly copy the content rather than
-   * paying the cost of exploding it, checksumming, and re-gzip.
+  /* See comments where we set this variable; this is implementing
+   * the --trusted-http/OSTREE_REPO_PULL_FLAGS_TRUSTED_HTTP flags.
    */
-  const gboolean mirroring_into_archive =
-    pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE;
-  const gboolean import_trusted = !verifying_bareuseronly &&
-    (pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_TRUSTED) > 0;
-  if (mirroring_into_archive && import_trusted)
+  if (pull_data->trusted_http_direct)
     {
-      gboolean have_object;
-      if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, checksum,
-                                   &have_object,
-                                   cancellable, error))
+      g_assert (!verifying_bareuseronly);
+      if (!_ostree_repo_commit_tmpf_final (pull_data->repo, checksum, objtype,
+                                           &tmpf, cancellable, error))
         goto out;
-
-      if (!have_object)
-        {
-          if (!_ostree_repo_commit_path_final (pull_data->repo, checksum, objtype,
-                                               &tmp_unlinker,
-                                               cancellable, error))
-            goto out;
-        }
       pull_data->n_fetched_content++;
     }
   else
     {
+      struct stat stbuf;
+      if (!glnx_fstat (tmpf.fd, &stbuf, error))
+        goto out;
       /* Non-mirroring path */
+      tmpf_input = g_unix_input_stream_new (glnx_steal_fd (&tmpf.fd), TRUE);
 
       /* If it appears corrupted, we'll delete it below */
-      if (!ostree_content_file_parse_at (TRUE, _ostree_fetcher_get_dfd (fetcher),
-                                         tmp_unlinker.path, FALSE,
-                                         &file_in, &file_info, &xattrs,
-                                         cancellable, error))
+      if (!ostree_content_stream_parse (TRUE, tmpf_input, stbuf.st_size, FALSE,
+                                        &file_in, &file_info, &xattrs,
+                                        cancellable, error))
         goto out;
 
-      /* Also, delete it now that we've opened it, we'll hold
-       * a reference to the fd.  If we fail to validate or write, then
-       * the temp space will be cleaned up.
-       */
-      ot_cleanup_unlinkat (&tmp_unlinker);
-
       if (verifying_bareuseronly)
         {
           if (!_ostree_validate_bareuseronly_mode_finfo (file_info, checksum, error))
@@ -1160,13 +1157,12 @@ meta_fetch_on_complete (GObject           *object,
   FetchObjectData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
   g_autoptr(GVariant) metadata = NULL;
-  g_auto(OtCleanupUnlinkat) tmp_unlinker = { _ostree_fetcher_get_dfd (fetcher), NULL };
+  g_auto(GLnxTmpfile) tmpf = { 0, };
   const char *checksum;
   g_autofree char *checksum_obj = NULL;
   OstreeObjectType objtype;
   g_autoptr(GError) local_error = NULL;
   GError **error = &local_error;
-  glnx_fd_close int fd = -1;
   gboolean free_fetch_data = TRUE;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -1174,7 +1170,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", checksum_obj,
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmp_unlinker.path, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
@@ -1185,7 +1181,7 @@ meta_fetch_on_complete (GObject           *object,
 
               /* Now that we've at least tried to fetch it, we can proceed to
                * scan/fetch the commit object */
-              g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (checksum));
+              g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
 
               if (!fetch_data->object_is_stored)
                 enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE, fetch_data->requested_ref);
@@ -1217,25 +1213,17 @@ meta_fetch_on_complete (GObject           *object,
   if (objtype == OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT)
     goto out;
 
-  if (!glnx_openat_rdonly (_ostree_fetcher_get_dfd (fetcher), tmp_unlinker.path, TRUE, &fd, error))
-    goto out;
-
-  /* Now delete it, keeping the fd open as the last reference; see comment in
-   * corresponding content fetch path.
-   */
-  ot_cleanup_unlinkat (&tmp_unlinker);
-
   if (fetch_data->is_detached_meta)
     {
-      if (!ot_util_variant_map_fd (fd, 0, G_VARIANT_TYPE ("a{sv}"),
-                                   FALSE, &metadata, error))
+      if (!ot_variant_read_fd (tmpf.fd, 0, G_VARIANT_TYPE ("a{sv}"),
+                               FALSE, &metadata, error))
         goto out;
 
       if (!ostree_repo_write_commit_detached_metadata (pull_data->repo, checksum, metadata,
                                                        pull_data->cancellable, error))
         goto out;
 
-      g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (checksum));
+      g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), g_steal_pointer (&metadata));
 
       if (!fetch_data->object_is_stored)
         enqueue_one_object_request (pull_data, checksum, objtype, fetch_data->path, FALSE, FALSE, fetch_data->requested_ref);
@@ -1244,13 +1232,40 @@ meta_fetch_on_complete (GObject           *object,
     }
   else
     {
-      if (!ot_util_variant_map_fd (fd, 0, ostree_metadata_variant_type (objtype),
-                                   FALSE, &metadata, error))
+      if (!ot_variant_read_fd (tmpf.fd, 0, ostree_metadata_variant_type (objtype),
+                               FALSE, &metadata, error))
         goto out;
 
-      /* Write the commitpartial file now while we're still fetching data */
+      /* For commit objects, compute the hash and check the GPG signature before
+       * writing to the repo, and also write the .commitpartial to say that
+       * we're still processing this commit.
+       */
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
+          /* Verify checksum */
+          OtChecksum hasher = { 0, };
+          ot_checksum_init (&hasher);
+          { g_autoptr(GBytes) bytes = g_variant_get_data_as_bytes (metadata);
+            ot_checksum_update_bytes (&hasher, bytes);
+          }
+          char hexdigest[OSTREE_SHA256_STRING_LEN+1];
+          ot_checksum_get_hexdigest (&hasher, hexdigest, sizeof (hexdigest));
+
+          if (!_ostree_compare_object_checksum (objtype, checksum, hexdigest, error))
+            goto out;
+
+          /* Do GPG verification. `detached_data` may be NULL if no detached
+           * metadata was found during pull; that's handled by
+           * gpg_verify_unwritten_commit(). If we ever change the pull code to
+           * not always fetch detached metadata, this bit will have to learn how
+           * to look up from the disk state as well, or insert the on-disk
+           * metadata into this hash.
+           */
+          GVariant *detached_data = g_hash_table_lookup (pull_data->fetched_detached_metadata, checksum);
+          if (!gpg_verify_unwritten_commit (pull_data, checksum, metadata, detached_data,
+                                            pull_data->cancellable, error))
+            goto out;
+
           if (!write_commitpartial_for (pull_data, checksum, error))
             goto out;
         }
@@ -1313,27 +1328,20 @@ static_deltapart_fetch_on_complete (GObject           *object,
   OstreeFetcher *fetcher = (OstreeFetcher *)object;
   FetchStaticDeltaData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
-  g_autofree char *temp_path = NULL;
+  g_auto(GLnxTmpfile) tmpf = { 0, };
   g_autoptr(GInputStream) in = NULL;
   g_autoptr(GVariant) part = NULL;
   g_autoptr(GError) local_error = NULL;
   GError **error = &local_error;
-  glnx_fd_close int fd = -1;
   gboolean free_fetch_data = TRUE;
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &tmpf, error))
     goto out;
 
-  if (!glnx_openat_rdonly (_ostree_fetcher_get_dfd (fetcher), temp_path, TRUE, &fd, error))
-    goto out;
-
-  /* From here on, if we fail to apply the delta, we'll re-fetch it */
-  if (!glnx_unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0, error))
-    goto out;
-
-  in = g_unix_input_stream_new (fd, FALSE);
+  /* Transfer ownership of the fd */
+  in = g_unix_input_stream_new (glnx_steal_fd (&tmpf.fd), TRUE);
 
   /* TODO - make async */
   if (!_ostree_static_delta_part_open (in, NULL, 0, fetch_data->expected_checksum,
@@ -1377,6 +1385,14 @@ process_verify_result (OtPullData            *pull_data,
   if (!ostree_gpg_verify_result_require_valid_signature (result, error))
     return FALSE;
 
+
+  /* We now check both *before* writing the commit, and after. Because the
+   * behavior used to be only verifiying after writing, we need to handle
+   * the case of "written but not verified". But we also don't want to check
+   * twice, as that'd result in duplicate signals.
+   */
+  g_hash_table_add (pull_data->gpg_verified_commits, g_strdup (checksum));
+
   return TRUE;
 }
 
@@ -1390,24 +1406,15 @@ gpg_verify_unwritten_commit (OtPullData         *pull_data,
 {
   if (pull_data->gpg_verify)
     {
-      g_autoptr(OstreeGpgVerifyResult) result = NULL;
+      /* Shouldn't happen, but see comment in process_verify_result() */
+      if (g_hash_table_contains (pull_data->gpg_verified_commits, checksum))
+        return TRUE;
+
       g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit);
-
-      if (!detached_metadata)
-        {
-          g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
-                       "Commit %s: no detached metadata found for GPG verification",
-                       checksum);
-          return FALSE;
-        }
-
-      result = _ostree_repo_gpg_verify_with_metadata (pull_data->repo,
-                                                      signed_data,
-                                                      detached_metadata,
-                                                      pull_data->remote_name,
-                                                      NULL, NULL,
-                                                      cancellable,
-                                                      error);
+      g_autoptr(OstreeGpgVerifyResult) result =
+        _ostree_repo_gpg_verify_with_metadata (pull_data->repo, signed_data,
+                                               detached_metadata, pull_data->remote_name,
+                                               NULL, NULL, cancellable, error);
       if (!process_verify_result (pull_data, checksum, result, error))
         return FALSE;
     }
@@ -1431,6 +1438,10 @@ static char *
 get_real_remote_repo_collection_id (OstreeRepo  *repo,
                                     const gchar *remote_name)
 {
+  /* remote_name == NULL can happen for pull-local */
+  if (!remote_name)
+    return NULL;
+
   g_autofree gchar *remote_collection_id = NULL;
   if (!ostree_repo_get_remote_option (repo, remote_name, "collection-id", NULL,
                                       &remote_collection_id, NULL) ||
@@ -1586,7 +1597,11 @@ scan_commit_object (OtPullData                 *pull_data,
                            GINT_TO_POINTER (depth));
     }
 
-  if (pull_data->gpg_verify)
+  /* See comment in process_verify_result() - we now gpg check before writing,
+   * but also ensure we've done it here if not already.
+   */
+  if (pull_data->gpg_verify &&
+      !g_hash_table_contains (pull_data->gpg_verified_commits, checksum))
     {
       g_autoptr(OstreeGpgVerifyResult) result = NULL;
 
@@ -1753,18 +1768,21 @@ queue_scan_one_metadata_object_c (OtPullData                *pull_data,
   ensure_idle_queued (pull_data);
 }
 
+/* Called out of the main loop to look at metadata objects which can have
+ * further references (commit, dirtree). See also idle_worker() which drives
+ * execution of this function.
+ */
 static gboolean
-scan_one_metadata_object_c (OtPullData                 *pull_data,
-                            const guchar                 *csum,
-                            OstreeObjectType            objtype,
-                            const char                 *path,
-                            guint                       recursion_depth,
-                            const OstreeCollectionRef  *ref,
-                            GCancellable               *cancellable,
-                            GError                    **error)
+scan_one_metadata_object (OtPullData                 *pull_data,
+                          const char                 *checksum,
+                          OstreeObjectType            objtype,
+                          const char                 *path,
+                          guint                       recursion_depth,
+                          const OstreeCollectionRef  *ref,
+                          GCancellable               *cancellable,
+                          GError                    **error)
 {
-  g_autofree char *tmp_checksum = ostree_checksum_from_bytes (csum);
-  g_autoptr(GVariant) object = ostree_object_name_serialize (tmp_checksum, objtype);
+  g_autoptr(GVariant) object = ostree_object_name_serialize (checksum, objtype);
 
   /* It may happen that we've already looked at this object (think shared
    * dirtree subtrees), if that's the case, we're done */
@@ -1774,7 +1792,7 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
   gboolean is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL;
   /* Determine if we already have the object */
   gboolean is_stored;
-  if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
+  if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored,
                                cancellable, error))
     return FALSE;
 
@@ -1784,19 +1802,19 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
           /* mark as partial to ensure we scan the commit below */
-          if (!write_commitpartial_for (pull_data, tmp_checksum, error))
+          if (!write_commitpartial_for (pull_data, checksum, error))
             return FALSE;
         }
 
       if (!_ostree_repo_import_object (pull_data->repo, pull_data->remote_repo_local,
-                                       objtype, tmp_checksum, pull_data->importflags,
+                                       objtype, checksum, pull_data->importflags,
                                        cancellable, error))
         return FALSE;
       /* The import API will fetch both the commit and detached metadata, so
        * add it to the hash to avoid re-fetching it below.
        */
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-        g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum));
+        g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
       pull_data->n_imported_metadata++;
       is_stored = TRUE;
       is_requested = TRUE;
@@ -1809,7 +1827,7 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
           OstreeRepo *refd_repo = pull_data->localcache_repos->pdata[i];
           gboolean localcache_repo_has_obj;
 
-          if (!ostree_repo_has_object (refd_repo, objtype, tmp_checksum,
+          if (!ostree_repo_has_object (refd_repo, objtype, checksum,
                                        &localcache_repo_has_obj, cancellable, error))
             return FALSE;
           if (!localcache_repo_has_obj)
@@ -1817,16 +1835,16 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
             {
               /* mark as partial to ensure we scan the commit below */
-              if (!write_commitpartial_for (pull_data, tmp_checksum, error))
+              if (!write_commitpartial_for (pull_data, checksum, error))
                 return FALSE;
             }
           if (!_ostree_repo_import_object (pull_data->repo, refd_repo,
-                                           objtype, tmp_checksum, pull_data->importflags,
+                                           objtype, checksum, pull_data->importflags,
                                            cancellable, error))
             return FALSE;
           /* See comment above */
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-            g_hash_table_add (pull_data->fetched_detached_metadata, g_strdup (tmp_checksum));
+            g_hash_table_insert (pull_data->fetched_detached_metadata, g_strdup (checksum), NULL);
           is_stored = TRUE;
           is_requested = TRUE;
           pull_data->n_imported_metadata++;
@@ -1841,18 +1859,18 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
       g_hash_table_add (pull_data->requested_metadata, g_variant_ref (object));
 
       do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
-      enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, do_fetch_detached, FALSE, ref);
+      enqueue_one_object_request (pull_data, checksum, objtype, path, do_fetch_detached, FALSE, ref);
     }
   else if (is_stored && objtype == OSTREE_OBJECT_TYPE_COMMIT)
     {
       /* Even though we already have the commit, we always try to (re)fetch the
        * detached metadata before scanning it, in case new signatures appear.
        * https://github.com/projectatomic/rpm-ostree/issues/630 */
-      if (!g_hash_table_contains (pull_data->fetched_detached_metadata, tmp_checksum))
-        enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, TRUE, TRUE, ref);
+      if (!g_hash_table_contains (pull_data->fetched_detached_metadata, checksum))
+        enqueue_one_object_request (pull_data, checksum, objtype, path, TRUE, TRUE, ref);
       else
         {
-          if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth, ref,
+          if (!scan_commit_object (pull_data, checksum, recursion_depth, ref,
                                    pull_data->cancellable, error))
             return FALSE;
 
@@ -1862,7 +1880,7 @@ scan_one_metadata_object_c (OtPullData                 *pull_data,
     }
   else if (is_stored && objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
     {
-      if (!scan_dirtree_object (pull_data, tmp_checksum, path, recursion_depth,
+      if (!scan_dirtree_object (pull_data, checksum, path, recursion_depth,
                                 pull_data->cancellable, error))
         return FALSE;
 
@@ -1975,6 +1993,8 @@ start_fetch (OtPullData *pull_data,
   else
     expected_max_size = 0;
 
+  if (!is_meta && pull_data->trusted_http_direct)
+    flags |= OSTREE_FETCHER_REQUEST_LINKABLE;
   _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
                                       obj_subpath, flags, expected_max_size,
                                       is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
@@ -2441,34 +2461,28 @@ on_superblock_fetched (GObject   *src,
     }
   else
     {
-      g_autofree gchar *delta = NULL;
-      g_autofree guchar *ret_csum = NULL;
-      guchar *summary_csum;
-      g_autoptr (GInputStream) summary_is = NULL;
       g_autoptr(GVariant) delta_superblock = NULL;
+      g_autofree gchar *delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
+      const guchar *expected_summary_digest = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
+      guint8 actual_summary_digest[OSTREE_SHA256_DIGEST_LEN];
 
-      summary_is = g_memory_input_stream_new_from_data (g_bytes_get_data (delta_superblock_data, NULL),
-                                                        g_bytes_get_size (delta_superblock_data),
-                                                        NULL);
-
-      if (!ot_gio_checksum_stream (summary_is, &ret_csum, pull_data->cancellable, error))
-        goto out;
-
-      delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
-      summary_csum = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
+      g_auto(OtChecksum) hasher = { 0, };
+      ot_checksum_init (&hasher);
+      ot_checksum_update_bytes (&hasher, delta_superblock_data);
+      ot_checksum_get_digest (&hasher, actual_summary_digest, sizeof (actual_summary_digest));
 
       /* At this point we've GPG verified the data, so in theory
        * could trust that they provided the right data, but let's
        * make this a hard error.
        */
-      if (pull_data->gpg_verify_summary && !summary_csum)
+      if (pull_data->gpg_verify_summary && !expected_summary_digest)
         {
           g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE,
                        "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
           goto out;
         }
 
-      if (summary_csum && memcmp (summary_csum, ret_csum, 32))
+      if (expected_summary_digest && memcmp (expected_summary_digest, actual_summary_digest, sizeof (actual_summary_digest)))
         {
           g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
           goto out;
@@ -2520,7 +2534,7 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
     return TRUE;
 
   const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig");
-  glnx_fd_close int prev_fd = -1;
+  glnx_autofd int prev_fd = -1;
   if (!ot_openat_ignore_enoent (self->cache_dir_fd, summary_cache_sig_file, &prev_fd, error))
     return FALSE;
   if (prev_fd < 0)
@@ -2533,7 +2547,7 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
   if (g_bytes_compare (old_sig_contents, summary_sig) == 0)
     {
       const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote);
-      glnx_fd_close int summary_fd = -1;
+      glnx_autofd int summary_fd = -1;
       GBytes *summary_data;
 
 
@@ -3031,7 +3045,7 @@ initiate_delta_request (OtPullData *pull_data,
 
   _ostree_fetcher_request_to_membuf (pull_data->fetcher,
                                      pull_data->content_mirrorlist,
-                                     delta_name, 0,
+                                     delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT,
                                      OSTREE_MAX_METADATA_SIZE,
                                      0, pull_data->cancellable,
                                      on_superblock_fetched, fdata);
@@ -3302,10 +3316,12 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   pull_data->ref_original_commits = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal,
                                                            (GDestroyNotify)NULL,
                                                            (GDestroyNotify)g_variant_unref);
+  pull_data->gpg_verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                                           (GDestroyNotify)g_free, NULL);
   pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
                                                        (GDestroyNotify)g_variant_unref, NULL);
   pull_data->fetched_detached_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
-                                                       (GDestroyNotify)g_free, NULL);
+                                                                (GDestroyNotify)g_free, (GDestroyNotify)variant_or_null_unref);
   pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                         (GDestroyNotify)g_free, NULL);
   pull_data->requested_fallback_content = g_hash_table_new_full (g_str_hash, g_str_equal,
@@ -3532,8 +3548,12 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_autofree char *first_scheme = _ostree_fetcher_uri_get_scheme (first_uri);
 
   /* NB: we don't support local mirrors in mirrorlists, so if this passes, it
-   * means that we're not using mirrorlists (see also fetch_mirrorlist()) */
-  if (g_str_equal (first_scheme, "file"))
+   * means that we're not using mirrorlists (see also fetch_mirrorlist())
+   * Also, we explicitly disable the "local repo" path if static deltas
+   * were explicitly requested to be required; this is going to happen
+   * most often for testing deltas without setting up a HTTP server.
+   */
+  if (g_str_equal (first_scheme, "file") && !pull_data->require_static_deltas)
     {
       g_autofree char *path = _ostree_fetcher_uri_get_path (first_uri);
       g_autoptr(GFile) remote_repo_path = g_file_new_for_path (path);
@@ -3583,6 +3603,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
        */
       if ((flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) == 0)
         pull_data->importflags |= _OSTREE_REPO_IMPORT_FLAGS_TRUSTED;
+
+      /* Shouldn't be referenced in this path, but just in case.  See below
+       * for more information.
+       */
+      pull_data->trusted_http_direct = FALSE;
     }
   else
     {
@@ -3593,6 +3618,18 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
        */
       if (flags & OSTREE_REPO_PULL_FLAGS_TRUSTED_HTTP)
         pull_data->importflags |= _OSTREE_REPO_IMPORT_FLAGS_TRUSTED;
+
+      const gboolean verifying_bareuseronly =
+        (pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY) > 0;
+      /* If we're mirroring and writing into an archive repo, and both checksum and
+       * bareuseronly are turned off, we can directly copy the content rather than
+       * paying the cost of exploding it, checksumming, and re-gzip.
+       */
+      const gboolean mirroring_into_archive =
+        pull_data->is_mirror && pull_data->repo->mode == OSTREE_REPO_MODE_ARCHIVE;
+      const gboolean import_trusted = !verifying_bareuseronly &&
+        (pull_data->importflags & _OSTREE_REPO_IMPORT_FLAGS_TRUSTED) > 0;
+      pull_data->trusted_http_direct = mirroring_into_archive && import_trusted;
     }
 
   /* We can't use static deltas if pulling into an archive repo. */
@@ -3824,6 +3861,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
       while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum))
         {
+          if (!ostree_validate_rev (ref_name, error))
+            goto out;
           g_hash_table_insert (requested_refs_to_fetch,
                                ostree_collection_ref_new (collection_id, ref_name),
                                (*checksum != '\0') ? g_strdup (checksum) : NULL);
@@ -4248,6 +4287,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_clear_pointer (&pull_data->fetched_detached_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->ref_original_commits, (GDestroyNotify) g_hash_table_unref);
+  g_clear_pointer (&pull_data->gpg_verified_commits, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);

src/libostree/ostree-repo-refs.c

@@ -150,7 +150,7 @@ find_ref_in_remotes (OstreeRepo         *self,
                      GError            **error)
 {
   g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
-  glnx_fd_close int ret_fd = -1;
+  glnx_autofd int ret_fd = -1;
 
   if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error))
     return FALSE;
@@ -158,7 +158,7 @@ find_ref_in_remotes (OstreeRepo         *self,
   while (TRUE)
     {
       struct dirent *dent = NULL;
-      glnx_fd_close int remote_dfd = -1;
+      glnx_autofd int remote_dfd = -1;
 
       if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, error))
         return FALSE;
@@ -234,7 +234,7 @@ resolve_refspec (OstreeRepo     *self,
 {
   __attribute__((unused)) GCancellable *cancellable = NULL;
   g_autofree char *ret_rev = NULL;
-  glnx_fd_close int target_fd = -1;
+  glnx_autofd int target_fd = -1;
 
   g_return_val_if_fail (ref != NULL, FALSE);
 
@@ -560,6 +560,14 @@ enumerate_refs_recurse (OstreeRepo    *repo,
       if (dent == NULL)
         break;
 
+      /* https://github.com/ostreedev/ostree/issues/1285
+       * Ignore any files that don't appear to be valid fragments; e.g.
+       * Red Hat has a tool that drops .rsync_info files into each
+       * directory it syncs.
+       **/
+      if (!_ostree_validate_ref_fragment (dent->d_name, NULL))
+        continue;
+
       g_string_append (base_path, dent->d_name);
 
       if (dent->d_type == DT_DIR)
@@ -637,7 +645,7 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
         {
           if (S_ISDIR (stbuf.st_mode))
             {
-              glnx_fd_close int base_fd = -1;
+              glnx_autofd int base_fd = -1;
               g_autoptr(GString) base_path = g_string_new ("");
               if (!cut_prefix)
                 g_string_printf (base_path, "%s/", ref_prefix);
@@ -652,7 +660,7 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
             }
           else
             {
-              glnx_fd_close int prefix_dfd = -1;
+              glnx_autofd int prefix_dfd = -1;
 
               if (!glnx_opendirat (self->repo_dir_fd, prefix_path, TRUE, &prefix_dfd, error))
                 return FALSE;
@@ -667,7 +675,7 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
     {
       g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
       g_autoptr(GString) base_path = g_string_new ("");
-      glnx_fd_close int refs_heads_dfd = -1;
+      glnx_autofd int refs_heads_dfd = -1;
 
       if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error))
         return FALSE;
@@ -687,7 +695,7 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
           while (TRUE)
             {
               struct dirent *dent;
-              glnx_fd_close int remote_dfd = -1;
+              glnx_autofd int remote_dfd = -1;
 
               if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
                 return FALSE;
@@ -992,7 +1000,7 @@ _ostree_repo_write_ref (OstreeRepo                 *self,
                         GCancellable               *cancellable,
                         GError                    **error)
 {
-  glnx_fd_close int dfd = -1;
+  glnx_autofd int dfd = -1;
 
   g_return_val_if_fail (remote == NULL || ref->collection_id == NULL, FALSE);
   g_return_val_if_fail (!(rev != NULL && alias != NULL), FALSE);
@@ -1016,7 +1024,7 @@ _ostree_repo_write_ref (OstreeRepo                 *self,
     }
   else if (remote == NULL && ref->collection_id != NULL)
     {
-      glnx_fd_close int refs_mirrors_dfd = -1;
+      glnx_autofd int refs_mirrors_dfd = -1;
 
       /* refs/mirrors might not exist in older repositories, so create it. */
       if (!glnx_shutil_mkdir_p_at_open (self->repo_dir_fd, "refs/mirrors", 0777,
@@ -1039,7 +1047,7 @@ _ostree_repo_write_ref (OstreeRepo                 *self,
     }
   else
     {
-      glnx_fd_close int refs_remotes_dfd = -1;
+      glnx_autofd int refs_remotes_dfd = -1;
 
       if (!glnx_opendirat (self->repo_dir_fd, "refs/remotes", TRUE,
                            &refs_remotes_dfd, error))
@@ -1212,7 +1220,7 @@ ostree_repo_list_collection_refs (OstreeRepo                 *self,
   if (main_collection_id != NULL &&
       (match_collection_id == NULL || g_strcmp0 (match_collection_id, main_collection_id) == 0))
     {
-      glnx_fd_close int refs_heads_dfd = -1;
+      glnx_autofd int refs_heads_dfd = -1;
 
       if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error))
         return FALSE;
@@ -1237,7 +1245,7 @@ ostree_repo_list_collection_refs (OstreeRepo                 *self,
       while (refs_dir_exists)
         {
           struct dirent *dent;
-          glnx_fd_close int subdir_fd = -1;
+          glnx_autofd int subdir_fd = -1;
           const gchar *current_collection_id;
           g_autofree gchar *remote_collection_id = NULL;
 

src/libostree/ostree-repo-static-delta-compilation.c

@@ -37,7 +37,14 @@
 
 #define CONTENT_SIZE_SIMILARITY_THRESHOLD_PERCENT (30)
 
+typedef enum {
+  DELTAOPT_FLAG_NONE = (1 << 0),
+  DELTAOPT_FLAG_DISABLE_BSDIFF = (1 << 1),
+  DELTAOPT_FLAG_VERBOSE = (1 << 2)
+} DeltaOpts;
+
 typedef struct {
+  guint64 compressed_size;
   guint64 uncompressed_size;
   GPtrArray *objects;
   GString *payload;
@@ -46,6 +53,8 @@ typedef struct {
   GPtrArray *modes;
   GHashTable *xattr_set; /* GVariant(ayay) -> offset */
   GPtrArray *xattrs;
+  GLnxTmpfile part_tmpf;
+  GVariant *header;
 } OstreeStaticDeltaPartBuilder;
 
 typedef struct {
@@ -60,13 +69,47 @@ typedef struct {
   guint n_bsdiff;
   guint n_fallback;
   gboolean swap_endian;
+  int parts_dfd;
+  DeltaOpts delta_opts;
 } OstreeStaticDeltaBuilder;
 
-typedef enum {
-  DELTAOPT_FLAG_NONE = (1 << 0),
-  DELTAOPT_FLAG_DISABLE_BSDIFF = (1 << 1),
-  DELTAOPT_FLAG_VERBOSE = (1 << 2)
-} DeltaOpts;
+/* Get an input stream for a GVariant */
+static GInputStream *
+variant_to_inputstream (GVariant             *variant)
+{
+  GMemoryInputStream *ret = (GMemoryInputStream*)
+    g_memory_input_stream_new_from_data (g_variant_get_data (variant),
+                                         g_variant_get_size (variant),
+                                         NULL);
+  g_object_set_data_full ((GObject*)ret, "ot-variant-data",
+                          g_variant_ref (variant), (GDestroyNotify) g_variant_unref);
+  return (GInputStream*)ret;
+}
+
+static GBytes *
+objtype_checksum_array_new (GPtrArray *objects)
+{
+  guint i;
+  GByteArray *ret = g_byte_array_new ();
+
+  for (i = 0; i < objects->len; i++)
+    {
+      GVariant *serialized_key = objects->pdata[i];
+      OstreeObjectType objtype;
+      const char *checksum;
+      guint8 csum[OSTREE_SHA256_DIGEST_LEN];
+      guint8 objtype_v;
+
+      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+      objtype_v = (guint8) objtype;
+
+      ostree_checksum_inplace_to_bytes (checksum, csum);
+
+      g_byte_array_append (ret, &objtype_v, 1);
+      g_byte_array_append (ret, csum, sizeof (csum));
+    }
+  return g_byte_array_free_to_bytes (ret);
+}
 
 static void
 ostree_static_delta_part_builder_unref (OstreeStaticDeltaPartBuilder *part_builder)
@@ -81,6 +124,9 @@ ostree_static_delta_part_builder_unref (OstreeStaticDeltaPartBuilder *part_build
   g_ptr_array_unref (part_builder->modes);
   g_hash_table_unref (part_builder->xattr_set);
   g_ptr_array_unref (part_builder->xattrs);
+  glnx_tmpfile_clear (&part_builder->part_tmpf);
+  if (part_builder->header)
+    g_variant_unref (part_builder->header);
   g_free (part_builder);
 }
 
@@ -162,10 +208,123 @@ xattr_chunk_equals (const void *one, const void *two)
   return memcmp (g_variant_get_data (v1), g_variant_get_data (v2), l1) == 0;
 }
 
+static gboolean
+finish_part (OstreeStaticDeltaBuilder *builder, GError **error)
+{
+  OstreeStaticDeltaPartBuilder *part_builder = builder->parts->pdata[builder->parts->len - 1];
+  g_autofree guchar *part_checksum = NULL;
+  g_autoptr(GBytes) objtype_checksum_array = NULL;
+  g_autoptr(GBytes) checksum_bytes = NULL;
+  g_autoptr(GOutputStream) part_temp_outstream = NULL;
+  g_autoptr(GInputStream) part_in = NULL;
+  g_autoptr(GInputStream) part_payload_in = NULL;
+  g_autoptr(GMemoryOutputStream) part_payload_out = NULL;
+  g_autoptr(GConverterOutputStream) part_payload_compressor = NULL;
+  g_autoptr(GConverter) compressor = NULL;
+  g_autoptr(GVariant) delta_part_content = NULL;
+  g_autoptr(GVariant) delta_part = NULL;
+  g_autoptr(GVariant) delta_part_header = NULL;
+  g_auto(GVariantBuilder) mode_builder = OT_VARIANT_BUILDER_INITIALIZER;
+  g_auto(GVariantBuilder) xattr_builder = OT_VARIANT_BUILDER_INITIALIZER;
+  guint8 compression_type_char;
+
+  g_variant_builder_init (&mode_builder, G_VARIANT_TYPE ("a(uuu)"));
+  g_variant_builder_init (&xattr_builder, G_VARIANT_TYPE ("aa(ayay)"));
+  guint j;
+
+  for (j = 0; j < part_builder->modes->len; j++)
+    g_variant_builder_add_value (&mode_builder, part_builder->modes->pdata[j]);
+
+  for (j = 0; j < part_builder->xattrs->len; j++)
+    g_variant_builder_add_value (&xattr_builder, part_builder->xattrs->pdata[j]);
+
+  {
+    g_autoptr(GBytes) payload_b;
+    g_autoptr(GBytes) operations_b;
+
+    payload_b = g_string_free_to_bytes (part_builder->payload);
+    part_builder->payload = NULL;
+
+    operations_b = g_string_free_to_bytes (part_builder->operations);
+    part_builder->operations = NULL;
+
+    delta_part_content = g_variant_new ("(a(uuu)aa(ayay)@ay@ay)",
+                                        &mode_builder, &xattr_builder,
+                                        ot_gvariant_new_ay_bytes (payload_b),
+                                        ot_gvariant_new_ay_bytes (operations_b));
+    g_variant_ref_sink (delta_part_content);
+  }
+
+  /* Hardcode xz for now */
+  compressor = (GConverter*)_ostree_lzma_compressor_new (NULL);
+  compression_type_char = 'x';
+  part_payload_in = variant_to_inputstream (delta_part_content);
+  part_payload_out = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+  part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new ((GOutputStream*)part_payload_out, compressor);
+
+  {
+    gssize n_bytes_written = g_output_stream_splice ((GOutputStream*)part_payload_compressor, part_payload_in,
+                                                     G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET | G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+                                                     NULL, error);
+    if (n_bytes_written < 0)
+      return FALSE;
+  }
+
+  g_clear_pointer (&delta_part_content, g_variant_unref);
+
+  { g_autoptr(GBytes) payload = g_memory_output_stream_steal_as_bytes (part_payload_out);
+    delta_part = g_variant_ref_sink (g_variant_new ("(y@ay)",
+                                                    compression_type_char,
+                                                    ot_gvariant_new_ay_bytes (payload)));
+  }
+
+  if (!glnx_open_tmpfile_linkable_at (builder->parts_dfd, ".", O_RDWR | O_CLOEXEC,
+                                      &part_builder->part_tmpf, error))
+    return FALSE;
+
+  part_temp_outstream = g_unix_output_stream_new (part_builder->part_tmpf.fd, FALSE);
+
+  part_in = variant_to_inputstream (delta_part);
+  if (!ot_gio_splice_get_checksum (part_temp_outstream, part_in,
+                                   &part_checksum,
+                                   NULL, error))
+    return FALSE;
+
+  checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
+  objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
+  delta_part_header = g_variant_new ("(u@aytt@ay)",
+                                     maybe_swap_endian_u32 (builder->swap_endian, OSTREE_DELTAPART_VERSION),
+                                     ot_gvariant_new_ay_bytes (checksum_bytes),
+                                     maybe_swap_endian_u64 (builder->swap_endian, (guint64) g_variant_get_size (delta_part)),
+                                     maybe_swap_endian_u64 (builder->swap_endian, part_builder->uncompressed_size),
+                                     ot_gvariant_new_ay_bytes (objtype_checksum_array));
+  g_variant_ref_sink (delta_part_header);
+
+  part_builder->header = g_variant_ref (delta_part_header);
+  part_builder->compressed_size = g_variant_get_size (delta_part);
+
+  if (builder->delta_opts & DELTAOPT_FLAG_VERBOSE)
+    {
+      g_printerr ("part %u n:%u compressed:%" G_GUINT64_FORMAT " uncompressed:%" G_GUINT64_FORMAT "\n",
+                  builder->parts->len, part_builder->objects->len,
+                  part_builder->compressed_size,
+                  part_builder->uncompressed_size);
+    }
+
+  return TRUE;
+}
+
 static OstreeStaticDeltaPartBuilder *
-allocate_part (OstreeStaticDeltaBuilder *builder)
+allocate_part (OstreeStaticDeltaBuilder *builder, GError **error)
 {
   OstreeStaticDeltaPartBuilder *part = g_new0 (OstreeStaticDeltaPartBuilder, 1);
+
+  if (builder->parts->len > 0)
+    {
+      if (!finish_part (builder, error))
+        return NULL;
+    }
+
   part->objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
   part->payload = g_string_new (NULL);
   part->operations = g_string_new (NULL);
@@ -221,31 +380,6 @@ write_unique_variant_chunk (OstreeStaticDeltaPartBuilder *current_part,
   return offset;
 }
 
-static GBytes *
-objtype_checksum_array_new (GPtrArray *objects)
-{
-  guint i;
-  GByteArray *ret = g_byte_array_new ();
-
-  for (i = 0; i < objects->len; i++)
-    {
-      GVariant *serialized_key = objects->pdata[i];
-      OstreeObjectType objtype;
-      const char *checksum;
-      guint8 csum[OSTREE_SHA256_DIGEST_LEN];
-      guint8 objtype_v;
-
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-      objtype_v = (guint8) objtype;
-
-      ostree_checksum_inplace_to_bytes (checksum, csum);
-
-      g_byte_array_append (ret, &objtype_v, 1);
-      g_byte_array_append (ret, csum, sizeof (csum));
-    }
-  return g_byte_array_free_to_bytes (ret);
-}
-
 static gboolean
 splice_stream_to_payload (OstreeStaticDeltaPartBuilder  *current_part,
                           GInputStream                  *istream,
@@ -338,7 +472,10 @@ process_one_object (OstreeRepo                       *repo,
   if (current_part->objects->len > 0 &&
       current_part->payload->len + content_size > builder->max_chunk_size_bytes)
     {
-      *current_part_val = current_part = allocate_part (builder);
+      current_part = allocate_part (builder, error);
+      if (current_part == NULL)
+        return FALSE;
+      *current_part_val = current_part;
     }
 
   guint64 compressed_size;
@@ -514,8 +651,8 @@ try_content_rollsum (OstreeRepo                       *repo,
 
   if (opts & DELTAOPT_FLAG_VERBOSE)
     {
-      g_printerr ("rollsum for %s; crcs=%u bufs=%u total=%u matchsize=%llu\n",
-                  to, matches->crcmatches,
+      g_printerr ("rollsum for %s -> %s; crcs=%u bufs=%u total=%u matchsize=%llu\n",
+                  from, to, matches->crcmatches,
                   matches->bufmatches,
                   matches->total, (unsigned long long)matches->match_size);
     }
@@ -577,7 +714,10 @@ process_one_rollsum (OstreeRepo                       *repo,
   if (current_part->objects->len > 0 &&
       current_part->payload->len > builder->max_chunk_size_bytes)
     {
-      *current_part_val = current_part = allocate_part (builder);
+      current_part = allocate_part (builder, error);
+      if (current_part == NULL)
+        return FALSE;
+      *current_part_val = current_part;
     }
 
   g_autoptr(GBytes) tmp_to = NULL;
@@ -692,7 +832,10 @@ process_one_bsdiff (OstreeRepo                       *repo,
   if (current_part->objects->len > 0 &&
       current_part->payload->len > builder->max_chunk_size_bytes)
     {
-      *current_part_val = current_part = allocate_part (builder);
+      current_part = allocate_part (builder, error);
+      if (current_part == NULL)
+        return FALSE;
+      *current_part_val = current_part;
     }
 
   g_autoptr(GBytes) tmp_from = NULL;
@@ -763,6 +906,20 @@ process_one_bsdiff (OstreeRepo                       *repo,
       _ostree_write_varuint64 (current_part->operations, current_part->payload->len);
       _ostree_write_varuint64 (current_part->operations, payload_size);
 
+      /* A bit too verbose to print by default...but leaving this #if 0'd out to
+       * use later. One approach I've been thinking about is to optionally
+       * output e.g. a JSON file as we build the deltas. Alternatively, we could
+       * try to reverse engineer things more in the "show" path, but that gets
+       * hard/messy as it's quite optimized for execution now.
+       */
+#if 0
+      g_printerr ("bspatch %s [%llu] → %s [%llu] bsdiff:%llu (%f)\n",
+                  bsdiff_content->from_checksum, (unsigned long long)tmp_from_len,
+                  to_checksum, (unsigned long long)tmp_to_len,
+                  (unsigned long long)payload_size,
+                  ((double)payload_size)/tmp_to_len);
+#endif
+
       g_string_append_len (current_part->payload, payload, payload_size);
     }
     g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
@@ -964,7 +1121,9 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
                   g_hash_table_size (modified_regfile_content));
     }
 
-  current_part = allocate_part (builder);
+  current_part = allocate_part (builder, error);
+  if (current_part == NULL)
+    return FALSE;
 
   /* Pack the metadata first */
   g_hash_table_iter_init (&hashiter, new_reachable_metadata);
@@ -1000,18 +1159,27 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
 
   /* Now do bsdiff'ed objects */
 
-  g_hash_table_iter_init (&hashiter, bsdiff_optimized_content_objects);
-  while (g_hash_table_iter_next (&hashiter, &key, &value))
+  const guint n_bsdiff = g_hash_table_size (bsdiff_optimized_content_objects);
+  if (n_bsdiff > 0)
     {
-      const char *checksum = key;
-      ContentBsdiff *bsdiff = value;
+      const guint mod = n_bsdiff / 10;
+      g_hash_table_iter_init (&hashiter, bsdiff_optimized_content_objects);
+      while (g_hash_table_iter_next (&hashiter, &key, &value))
+        {
+          const char *checksum = key;
+          ContentBsdiff *bsdiff = value;
 
-      if (!process_one_bsdiff (repo, builder, &current_part,
-                               checksum, bsdiff,
-                               cancellable, error))
-        return FALSE;
+          if (opts & DELTAOPT_FLAG_VERBOSE &&
+              (mod == 0 || builder->n_bsdiff % mod == 0))
+            g_printerr ("processing bsdiff: [%u/%u]\n", builder->n_bsdiff, n_bsdiff);
 
-      builder->n_bsdiff++;
+          if (!process_one_bsdiff (repo, builder, &current_part,
+                                   checksum, bsdiff,
+                                   cancellable, error))
+            return FALSE;
+
+          builder->n_bsdiff++;
+        }
     }
 
   /* Scan for large objects, so we can fall back to plain HTTP-based
@@ -1080,6 +1248,9 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
         return FALSE;
     }
 
+  if (!finish_part (builder, error))
+    return FALSE;
+
   return TRUE;
 }
 
@@ -1186,24 +1357,24 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   guint min_fallback_size;
   guint max_bsdiff_size;
   guint max_chunk_size;
-  g_auto(GVariantBuilder) metadata_builder = OT_VARIANT_BUILDER_INITIALIZER;
   DeltaOpts delta_opts = DELTAOPT_FLAG_NONE;
   guint64 total_compressed_size = 0;
   guint64 total_uncompressed_size = 0;
   g_autoptr(GVariantBuilder) part_headers = NULL;
   g_autoptr(GPtrArray) part_temp_paths = NULL;
-  g_autoptr(GVariant) delta_descriptor = NULL;
   g_autoptr(GVariant) to_commit = NULL;
   const char *opt_filename;
   g_autofree char *descriptor_name = NULL;
-  glnx_fd_close int descriptor_dfd = -1;
+  glnx_autofd int descriptor_dfd = -1;
   g_autoptr(GVariant) fallback_headers = NULL;
   g_autoptr(GVariant) detached = NULL;
   gboolean inline_parts;
   guint endianness = G_BYTE_ORDER;
-  glnx_fd_close int tmp_dfd = -1;
+  glnx_autofd int tmp_dfd = -1;
   builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
   builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+  g_auto(GLnxTmpfile) descriptor_tmpf = { 0, };
+  g_autoptr(OtVariantBuilder) descriptor_builder = NULL;
 
   if (!g_variant_lookup (params, "min-fallback-size", "u", &min_fallback_size))
     min_fallback_size = 4;
@@ -1245,45 +1416,6 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                  &to_commit, error))
     goto out;
 
-  /* Ignore optimization flags */
-  if (!generate_delta_lowlatency (self, from, to, delta_opts, &builder,
-                                  cancellable, error))
-    goto out;
-
-  /* NOTE: Add user-supplied metadata first.  This is used by at least
-   * xdg-app as a way to provide MIME content sniffing, since the
-   * metadata appears first in the file.
-   */
-  g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
-  if (metadata != NULL)
-    {
-      GVariantIter iter;
-      GVariant *item;
-
-      g_variant_iter_init (&iter, metadata);
-      while ((item = g_variant_iter_next_value (&iter)))
-        {
-          g_variant_builder_add (&metadata_builder, "@{sv}", item);
-          g_variant_unref (item);
-        }
-    }
-
-  { guint8 endianness_char;
-
-    switch (endianness)
-      {
-      case G_LITTLE_ENDIAN:
-        endianness_char = 'l';
-        break;
-      case G_BIG_ENDIAN:
-        endianness_char = 'B';
-        break;
-      default:
-        g_assert_not_reached ();
-      }
-    g_variant_builder_add (&metadata_builder, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char));
-  }
-
   if (opt_filename)
     {
       g_autofree char *dnbuf = g_strdup (opt_filename);
@@ -1301,119 +1433,13 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
         }
     }
 
-  part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT));
-  part_temp_paths = g_ptr_array_new_with_free_func ((GDestroyNotify)glnx_tmpfile_clear);
-  for (i = 0; i < builder.parts->len; i++)
-    {
-      OstreeStaticDeltaPartBuilder *part_builder = builder.parts->pdata[i];
-      g_autoptr(GBytes) payload_b;
-      g_autoptr(GBytes) operations_b;
-      g_autofree guchar *part_checksum = NULL;
-      g_autoptr(GBytes) objtype_checksum_array = NULL;
-      g_autoptr(GBytes) checksum_bytes = NULL;
-      g_autoptr(GOutputStream) part_temp_outstream = NULL;
-      g_autoptr(GInputStream) part_in = NULL;
-      g_autoptr(GInputStream) part_payload_in = NULL;
-      g_autoptr(GMemoryOutputStream) part_payload_out = NULL;
-      g_autoptr(GConverterOutputStream) part_payload_compressor = NULL;
-      g_autoptr(GConverter) compressor = NULL;
-      g_autoptr(GVariant) delta_part_content = NULL;
-      g_autoptr(GVariant) delta_part = NULL;
-      g_autoptr(GVariant) delta_part_header = NULL;
-      g_auto(GVariantBuilder) mode_builder = OT_VARIANT_BUILDER_INITIALIZER;
-      g_auto(GVariantBuilder) xattr_builder = OT_VARIANT_BUILDER_INITIALIZER;
-      guint8 compression_type_char;
+  builder.parts_dfd = tmp_dfd;
+  builder.delta_opts = delta_opts;
 
-      g_variant_builder_init (&mode_builder, G_VARIANT_TYPE ("a(uuu)"));
-      g_variant_builder_init (&xattr_builder, G_VARIANT_TYPE ("aa(ayay)"));
-      { guint j;
-        for (j = 0; j < part_builder->modes->len; j++)
-          g_variant_builder_add_value (&mode_builder, part_builder->modes->pdata[j]);
-
-        for (j = 0; j < part_builder->xattrs->len; j++)
-          g_variant_builder_add_value (&xattr_builder, part_builder->xattrs->pdata[j]);
-      }
-
-      payload_b = g_string_free_to_bytes (part_builder->payload);
-      part_builder->payload = NULL;
-
-      operations_b = g_string_free_to_bytes (part_builder->operations);
-      part_builder->operations = NULL;
-      /* FIXME - avoid duplicating memory here */
-      delta_part_content = g_variant_new ("(a(uuu)aa(ayay)@ay@ay)",
-                                          &mode_builder, &xattr_builder,
-                                          ot_gvariant_new_ay_bytes (payload_b),
-                                          ot_gvariant_new_ay_bytes (operations_b));
-      g_variant_ref_sink (delta_part_content);
-
-      /* Hardcode xz for now */
-      compressor = (GConverter*)_ostree_lzma_compressor_new (NULL);
-      compression_type_char = 'x';
-      part_payload_in = ot_variant_read (delta_part_content);
-      part_payload_out = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-      part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new ((GOutputStream*)part_payload_out, compressor);
-
-      {
-        gssize n_bytes_written = g_output_stream_splice ((GOutputStream*)part_payload_compressor, part_payload_in,
-                                                         G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET | G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                                                         cancellable, error);
-        if (n_bytes_written < 0)
-          goto out;
-      }
-
-      /* FIXME - avoid duplicating memory here */
-      { g_autoptr(GBytes) payload = g_memory_output_stream_steal_as_bytes (part_payload_out);
-        delta_part = g_variant_ref_sink (g_variant_new ("(y@ay)",
-                                                        compression_type_char,
-                                                        ot_gvariant_new_ay_bytes (payload)));
-      }
-
-      if (inline_parts)
-        {
-          g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
-          g_variant_builder_add (&metadata_builder, "{sv}", part_relpath, delta_part);
-        }
-      else
-        {
-          GLnxTmpfile *part_tmpf = g_new0 (GLnxTmpfile, 1);
-
-          if (!glnx_open_tmpfile_linkable_at (tmp_dfd, ".", O_WRONLY | O_CLOEXEC,
-                                              part_tmpf, error))
-            goto out;
-
-          /* Transfer tempfile ownership */
-          part_temp_outstream = g_unix_output_stream_new (part_tmpf->fd, FALSE);
-          g_ptr_array_add (part_temp_paths, g_steal_pointer (&part_tmpf));
-        }
-
-      part_in = ot_variant_read (delta_part);
-      if (!ot_gio_splice_get_checksum (part_temp_outstream, part_in,
-                                       &part_checksum,
-                                       cancellable, error))
-        goto out;
-
-      checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
-      objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
-      delta_part_header = g_variant_new ("(u@aytt@ay)",
-                                         maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION),
-                                         ot_gvariant_new_ay_bytes (checksum_bytes),
-                                         maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)),
-                                         maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size),
-                                         ot_gvariant_new_ay_bytes (objtype_checksum_array));
-
-      g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
-
-      total_compressed_size += g_variant_get_size (delta_part);
-      total_uncompressed_size += part_builder->uncompressed_size;
-
-      if (delta_opts & DELTAOPT_FLAG_VERBOSE)
-        {
-          g_printerr ("part %u n:%u compressed:%" G_GUINT64_FORMAT " uncompressed:%" G_GUINT64_FORMAT "\n",
-                      i, part_builder->objects->len,
-                      (guint64)g_variant_get_size (delta_part),
-                      part_builder->uncompressed_size);
-        }
-    }
+  /* Ignore optimization flags */
+  if (!generate_delta_lowlatency (self, from, to, delta_opts, &builder,
+                                  cancellable, error))
+    goto out;
 
   if (opt_filename)
     {
@@ -1438,22 +1464,90 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       descriptor_name = g_strdup (basename (descriptor_relpath));
     }
 
-  for (i = 0; i < part_temp_paths->len; i++)
-    {
-      g_autofree char *partstr = g_strdup_printf ("%u", i);
-      /* Take ownership of the path/fd here */
-      g_auto(GLnxTmpfile) tmpf = *((GLnxTmpfile*)part_temp_paths->pdata[i]);
-      g_clear_pointer (&(part_temp_paths->pdata[i]), g_free);
+  if (!glnx_open_tmpfile_linkable_at (descriptor_dfd, ".", O_WRONLY | O_CLOEXEC,
+                                      &descriptor_tmpf, error))
+    goto out;
 
-      if (fchmod (tmpf.fd, 0644) < 0)
+  descriptor_builder = ot_variant_builder_new (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), descriptor_tmpf.fd);
+
+  /* Open the metadata dict */
+  if (!ot_variant_builder_open (descriptor_builder, G_VARIANT_TYPE ("a{sv}"), error))
+    goto out;
+
+  /* NOTE: Add user-supplied metadata first.  This is used by at least
+   * flatpak as a way to provide MIME content sniffing, since the
+   * metadata appears first in the file.
+   */
+  if (metadata != NULL)
+    {
+      GVariantIter iter;
+      GVariant *item;
+
+      g_variant_iter_init (&iter, metadata);
+      while ((item = g_variant_iter_next_value (&iter)))
         {
-          glnx_set_error_from_errno (error);
-          goto out;
+          if (!ot_variant_builder_add_value (descriptor_builder, item, error))
+            goto out;
+          g_variant_unref (item);
+        }
+    }
+
+  { guint8 endianness_char;
+
+    switch (endianness)
+      {
+      case G_LITTLE_ENDIAN:
+        endianness_char = 'l';
+        break;
+      case G_BIG_ENDIAN:
+        endianness_char = 'B';
+        break;
+      default:
+        g_assert_not_reached ();
+      }
+    if (!ot_variant_builder_add (descriptor_builder, error, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char)))
+      goto out;
+  }
+
+  part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT));
+  part_temp_paths = g_ptr_array_new_with_free_func ((GDestroyNotify)glnx_tmpfile_clear);
+  for (i = 0; i < builder.parts->len; i++)
+    {
+      OstreeStaticDeltaPartBuilder *part_builder = builder.parts->pdata[i];
+
+      if (inline_parts)
+        {
+          g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
+
+          lseek (part_builder->part_tmpf.fd, 0, SEEK_SET);
+
+          if (!ot_variant_builder_open (descriptor_builder, G_VARIANT_TYPE ("{sv}"), error) ||
+              !ot_variant_builder_add (descriptor_builder, error, "s", part_relpath) ||
+              !ot_variant_builder_open (descriptor_builder, G_VARIANT_TYPE ("v"), error) ||
+              !ot_variant_builder_add_from_fd (descriptor_builder, G_VARIANT_TYPE ("(yay)"), part_builder->part_tmpf.fd, part_builder->compressed_size, error) ||
+              !ot_variant_builder_close (descriptor_builder, error) ||
+              !ot_variant_builder_close (descriptor_builder, error))
+            goto out;
+        }
+      else
+        {
+          g_autofree char *partstr = g_strdup_printf ("%u", i);
+
+          if (fchmod (part_builder->part_tmpf.fd, 0644) < 0)
+            {
+              glnx_set_error_from_errno (error);
+              goto out;
+            }
+
+          if (!glnx_link_tmpfile_at (&part_builder->part_tmpf, GLNX_LINK_TMPFILE_REPLACE,
+                                     descriptor_dfd, partstr, error))
+            goto out;
         }
 
-      if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE,
-                                 descriptor_dfd, partstr, error))
-        goto out;
+      g_variant_builder_add_value (part_headers, g_variant_ref (part_builder->header));
+
+      total_compressed_size += part_builder->compressed_size;
+      total_uncompressed_size += part_builder->uncompressed_size;
     }
 
   if (!get_fallback_headers (self, &builder, &fallback_headers,
@@ -1466,9 +1560,14 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   if (detached)
     {
       g_autofree char *detached_key = _ostree_get_relative_static_delta_path (from, to, "commitmeta");
-      g_variant_builder_add (&metadata_builder, "{sv}", detached_key, detached);
+      if (!ot_variant_builder_add (descriptor_builder, error, "{sv}", detached_key, detached))
+        goto out;
     }
 
+  /* Close metadata dict */
+  if (!ot_variant_builder_close (descriptor_builder, error))
+    goto out;
+
   /* Generate OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
   {
     GDateTime *now = g_date_time_new_now_utc ();
@@ -1477,17 +1576,26 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     /* floating */ GVariant *to_csum_v =
       ostree_checksum_to_bytes_v (to);
 
-    delta_descriptor = g_variant_ref_sink (g_variant_new ("(@a{sv}t@ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "@ay"
-                                                          "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
-                                                          "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
-                                                          g_variant_builder_end (&metadata_builder),
-                                                          GUINT64_TO_BE (g_date_time_to_unix (now)),
-                                                          from_csum_v,
-                                                          to_csum_v,
-                                                          to_commit,
-                                                          ot_gvariant_new_bytearray ((guchar*)"", 0),
-                                                          part_headers,
-                                                          fallback_headers));
+
+    if (!ot_variant_builder_add (descriptor_builder, error, "t",
+                                 GUINT64_TO_BE (g_date_time_to_unix (now))) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       from_csum_v, error) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       to_csum_v, error) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       to_commit, error) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       ot_gvariant_new_bytearray ((guchar*)"", 0), error) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       g_variant_builder_end (part_headers), error) ||
+        !ot_variant_builder_add_value (descriptor_builder,
+                                       fallback_headers, error))
+      goto out;
+
+    if (!ot_variant_builder_end (descriptor_builder, error))
+      goto out;
+
     g_date_time_unref (now);
   }
 
@@ -1503,10 +1611,14 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       g_printerr ("bsdiff=%u objects\n", builder.n_bsdiff);
     }
 
-  if (!glnx_file_replace_contents_at (descriptor_dfd, descriptor_name,
-                                      g_variant_get_data (delta_descriptor),
-                                      g_variant_get_size (delta_descriptor),
-                                      0, cancellable, error))
+  if (fchmod (descriptor_tmpf.fd, 0644) < 0)
+    {
+      glnx_set_error_from_errno (error);
+      goto out;
+    }
+
+  if (!glnx_link_tmpfile_at (&descriptor_tmpf, GLNX_LINK_TMPFILE_REPLACE,
+                             descriptor_dfd, descriptor_name, error))
     goto out;
 
   ret = TRUE;

src/libostree/ostree-repo-static-delta-core.c

@@ -224,7 +224,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
   const char *dir_or_file_path = gs_file_get_path_cached (dir_or_file);
 
   /* First, try opening it as a directory */
-  glnx_fd_close int dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE);
+  glnx_autofd int dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE);
   if (dfd < 0)
     {
       if (errno != ENOTDIR)
@@ -241,13 +241,13 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
   else
     basename = g_strdup ("superblock");
 
-  glnx_fd_close int meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC);
+  glnx_autofd int meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC);
   if (meta_fd < 0)
     return glnx_throw_errno_prefix (error, "openat(%s)", basename);
 
   g_autoptr(GVariant) meta = NULL;
-  if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
-                               FALSE, &meta, error))
+  if (!ot_variant_read_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
+                           FALSE, &meta, error))
     return FALSE;
 
   /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
@@ -377,7 +377,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
       else
         {
           g_autofree char *relpath = g_strdup_printf ("%u", i); /* TODO avoid malloc here */
-          glnx_fd_close int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC);
+          glnx_autofd int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC);
           if (part_fd < 0)
             return glnx_throw_errno_prefix (error, "Opening deltapart '%s'", relpath);
 
@@ -448,8 +448,8 @@ _ostree_static_delta_part_open (GInputStream   *part_in,
           int part_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)part_in);
 
           /* No compression, no checksums - a fast path */
-          if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
-                                       trusted, &ret_part, error))
+          if (!ot_variant_read_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
+                                   trusted, &ret_part, error))
             return FALSE;
         }
       else
@@ -525,7 +525,7 @@ show_one_part (OstreeRepo                    *self,
   g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
            i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize);
 
-  glnx_fd_close gint part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC);
+  glnx_autofd int part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC);
   if (part_fd < 0)
     return glnx_throw_errno_prefix (error, "openat(%s)", part_path);
   g_autoptr(GInputStream) part_in = g_unix_input_stream_new (part_fd, FALSE);
@@ -767,9 +767,12 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
 
   superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
 
-  if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path,
-                               (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
-                               OT_VARIANT_MAP_TRUSTED, &delta_superblock, error))
+  glnx_autofd int superblock_fd = -1;
+  if (!glnx_openat_rdonly (self->repo_dir_fd, superblock_path, TRUE, &superblock_fd, error))
+    return FALSE;
+  if (!ot_variant_read_fd (superblock_fd, 0,
+                           (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
+                           TRUE, &delta_superblock, error))
     return FALSE;
 
   g_print ("Delta: %s\n", delta_id);

src/libostree/ostree-repo-static-delta-processing.c

@@ -58,7 +58,7 @@ typedef struct {
   GLnxTmpfile      tmpf;
   guint64          content_size;
   GOutputStream   *content_out;
-  GChecksum       *content_checksum;
+  OtChecksum       content_checksum;
   char             checksum[OSTREE_SHA256_STRING_LEN+1];
   char             *read_source_object;
   int               read_source_fd;
@@ -229,6 +229,9 @@ _ostree_static_delta_part_execute (OstreeRepo      *repo,
       state->oplen--;
       state->opdata++;
 
+      if (g_cancellable_set_error_if_cancelled (cancellable, error))
+        goto out;
+
       switch (opcode)
         {
         case OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE:
@@ -277,7 +280,7 @@ _ostree_static_delta_part_execute (OstreeRepo      *repo,
  out:
   glnx_tmpfile_clear (&state->tmpf);
   g_clear_object (&state->content_out);
-  g_clear_pointer (&state->content_checksum, g_checksum_free);
+  ot_checksum_clear (&state->content_checksum);
   return ret;
 }
 
@@ -385,8 +388,8 @@ content_out_write (OstreeRepo                 *repo,
 {
   gsize bytes_written;
 
-  if (state->content_checksum)
-    g_checksum_update (state->content_checksum, buf, len);
+  if (state->content_checksum.initialized)
+    ot_checksum_update (&state->content_checksum, buf, len);
 
   /* Ignore bytes_written since we discard partial content */
   if (!g_output_stream_write_all (state->content_out,
@@ -501,14 +504,10 @@ handle_untrusted_content_checksum (OstreeRepo                 *repo,
                                    GError                    **error)
 {
   g_autoptr(GFileInfo) finfo = _ostree_mode_uidgid_to_gfileinfo (state->mode, state->uid, state->gid);
-  g_autoptr(GVariant) header = _ostree_file_header_new (finfo, state->xattrs);
+  g_autoptr(GBytes) header = _ostree_file_header_new (finfo, state->xattrs);
 
-  state->content_checksum = g_checksum_new (G_CHECKSUM_SHA256);
-
-  gsize bytes_written;
-  if (!_ostree_write_variant_with_size (NULL, header, 0, &bytes_written, state->content_checksum,
-                                        cancellable, error))
-    return FALSE;
+  ot_checksum_init (&state->content_checksum);
+  ot_checksum_update_bytes (&state->content_checksum, header);
 
   return TRUE;
 }
@@ -582,18 +581,20 @@ dispatch_open_splice_and_close (OstreeRepo                 *repo,
 
       /* Fast path for regular files to bare repositories */
       if (S_ISREG (state->mode) &&
-          (repo->mode == OSTREE_REPO_MODE_BARE ||
-           repo->mode == OSTREE_REPO_MODE_BARE_USER))
+          _ostree_repo_mode_is_bare (repo->mode))
         {
-          if (!_ostree_repo_open_content_bare (repo, state->checksum,
-                                               state->content_size,
-                                               &state->tmpf,
-                                               &state->have_obj,
-                                               cancellable, error))
+          if (!ostree_repo_has_object (repo, OSTREE_OBJECT_TYPE_FILE, state->checksum,
+                                       &state->have_obj, cancellable, error))
             goto out;
 
           if (!state->have_obj)
             {
+              if (!_ostree_repo_open_content_bare (repo, state->checksum,
+                                                   state->content_size,
+                                                   &state->tmpf,
+                                                   cancellable, error))
+                goto out;
+
               state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
               if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
                 goto out;
@@ -684,14 +685,19 @@ dispatch_open (OstreeRepo                 *repo,
   if (state->stats_only)
     return TRUE; /* Early return */
 
-  if (!_ostree_repo_open_content_bare (repo, state->checksum,
-                                       state->content_size,
-                                       &state->tmpf,
-                                       &state->have_obj,
-                                       cancellable, error))
+  if (!ostree_repo_has_object (repo, OSTREE_OBJECT_TYPE_FILE, state->checksum,
+                               &state->have_obj, cancellable, error))
     return FALSE;
+
   if (!state->have_obj)
-    state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
+    {
+      if (!_ostree_repo_open_content_bare (repo, state->checksum,
+                                           state->content_size,
+                                           &state->tmpf,
+                                           cancellable, error))
+        return FALSE;
+      state->content_out = g_unix_output_stream_new (state->tmpf.fd, FALSE);
+    }
 
   if (!handle_untrusted_content_checksum (repo, state, cancellable, error))
     return FALSE;
@@ -705,7 +711,7 @@ dispatch_write (OstreeRepo                 *repo,
                GCancellable               *cancellable,
                GError                    **error)
 {
-  GLNX_AUTO_PREFIX_ERROR("opcode open-splice-and-close", error);
+  GLNX_AUTO_PREFIX_ERROR("opcode write", error);
   guint64 content_size;
   guint64 content_offset;
 
@@ -721,15 +727,13 @@ dispatch_write (OstreeRepo                 *repo,
     {
       if (state->read_source_fd != -1)
         {
-          if (lseek (state->read_source_fd, content_offset, SEEK_SET) == -1)
-            return glnx_throw_errno_prefix (error, "lseek");
           while (content_size > 0)
             {
               char buf[4096];
               gssize bytes_read;
 
               do
-                bytes_read = read (state->read_source_fd, buf, MIN(sizeof(buf), content_size));
+                bytes_read = pread (state->read_source_fd, buf, MIN(sizeof(buf), content_size), content_offset);
               while (G_UNLIKELY (bytes_read == -1 && errno == EINTR));
               if (bytes_read == -1)
                 return glnx_throw_errno_prefix (error, "read");
@@ -741,6 +745,7 @@ dispatch_write (OstreeRepo                 *repo,
                 return FALSE;
 
               content_size -= bytes_read;
+              content_offset += bytes_read;
             }
         }
       else
@@ -766,11 +771,7 @@ dispatch_set_read_source (OstreeRepo                 *repo,
   GLNX_AUTO_PREFIX_ERROR("opcode set-read-source", error);
   guint64 source_offset;
 
-  if (state->read_source_fd != -1)
-    {
-      (void) close (state->read_source_fd);
-      state->read_source_fd = -1;
-    }
+  glnx_close_fd (&state->read_source_fd);
 
   if (!read_varuint64 (state, &source_offset, error))
     return FALSE;
@@ -803,12 +804,7 @@ dispatch_unset_read_source (OstreeRepo                 *repo,
   if (state->stats_only)
     return TRUE; /* Early return */
 
-  if (state->read_source_fd != -1)
-    {
-      (void) close (state->read_source_fd);
-      state->read_source_fd = -1;
-    }
-
+  glnx_close_fd (&state->read_source_fd);
   g_clear_pointer (&state->read_source_object, g_free);
 
   return TRUE;
@@ -820,16 +816,17 @@ dispatch_close (OstreeRepo                 *repo,
                 GCancellable               *cancellable,
                 GError                    **error)
 {
-  GLNX_AUTO_PREFIX_ERROR("opcode open-splice-and-close", error);
+  GLNX_AUTO_PREFIX_ERROR("opcode close", error);
 
   if (state->content_out)
     {
       if (!g_output_stream_flush (state->content_out, cancellable, error))
         return FALSE;
 
-      if (state->content_checksum)
+      if (state->content_checksum.initialized)
         {
-          const char *actual_checksum = g_checksum_get_string (state->content_checksum);
+          char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+          ot_checksum_get_hexdigest (&state->content_checksum, actual_checksum, sizeof (actual_checksum));
 
           if (strcmp (actual_checksum, state->checksum) != 0)
             return glnx_throw (error, "Corrupted object %s (actual checksum is %s)",
@@ -848,7 +845,7 @@ dispatch_close (OstreeRepo                 *repo,
     return FALSE;
 
   g_clear_pointer (&state->xattrs, g_variant_unref);
-  g_clear_pointer (&state->content_checksum, g_checksum_free);
+  ot_checksum_clear (&state->content_checksum);
 
   state->checksum_index++;
   state->output_target = NULL;

src/libostree/ostree-repo.c

@@ -488,18 +488,13 @@ ostree_repo_finalize (GObject *object)
   g_free (self->stagedir_prefix);
   g_clear_object (&self->repodir_fdrel);
   g_clear_object (&self->repodir);
-  if (self->repo_dir_fd != -1)
-    (void) close (self->repo_dir_fd);
+  glnx_close_fd (&self->repo_dir_fd);
   glnx_tmpdir_unset (&self->commit_stagedir);
   glnx_release_lock_file (&self->commit_stagedir_lock);
-  if (self->tmp_dir_fd != -1)
-    (void) close (self->tmp_dir_fd);
-  if (self->cache_dir_fd != -1)
-    (void) close (self->cache_dir_fd);
-  if (self->objects_dir_fd != -1)
-    (void) close (self->objects_dir_fd);
-  if (self->uncompressed_objects_dir_fd != -1)
-    (void) close (self->uncompressed_objects_dir_fd);
+  glnx_close_fd (&self->tmp_dir_fd);
+  glnx_close_fd (&self->cache_dir_fd);
+  glnx_close_fd (&self->objects_dir_fd);
+  glnx_close_fd (&self->uncompressed_objects_dir_fd);
   g_clear_object (&self->sysroot_dir);
   g_weak_ref_clear (&self->sysroot);
   g_free (self->remotes_config_dir);
@@ -734,7 +729,7 @@ ostree_repo_open_at (int           dfd,
                      GCancellable *cancellable,
                      GError      **error)
 {
-  glnx_fd_close int repo_dfd = -1;
+  glnx_autofd int repo_dfd = -1;
   if (!glnx_opendirat (dfd, path, TRUE, &repo_dfd, error))
     return NULL;
 
@@ -1463,7 +1458,7 @@ ostree_repo_remote_gpg_import (OstreeRepo         *self,
   gpgme_import_status_t import_status;
   g_autofree char *source_tmp_dir = NULL;
   g_autofree char *target_tmp_dir = NULL;
-  glnx_fd_close int target_temp_fd = -1;
+  glnx_autofd int target_temp_fd = -1;
   g_autoptr(GPtrArray) keys = NULL;
   struct stat stbuf;
   gpgme_error_t gpg_error;
@@ -1596,7 +1591,7 @@ ostree_repo_remote_gpg_import (OstreeRepo         *self,
     }
   else if (errno == ENOENT)
     {
-      glnx_fd_close int fd = -1;
+      glnx_autofd int fd = -1;
 
       /* Create an empty pubring.gpg file prior to importing keys.  This
        * prevents gpg2 from creating a pubring.kbx file in the new keybox
@@ -1711,6 +1706,9 @@ out:
  * @NULL.  Likewise if the summary file is not signed, @out_signatures is
  * set to @NULL.  In either case the function still returns %TRUE.
  *
+ * This method does not verify the signature of the downloaded summary file.
+ * Use ostree_repo_verify_summary() for that.
+ *
  * Parse the summary data into a #GVariant using g_variant_new_from_bytes()
  * with #OSTREE_SUMMARY_GVARIANT_FORMAT as the format string.
  *
@@ -1812,7 +1810,7 @@ repo_create_at_internal (int             dfd,
       return FALSE;
     if (errno == 0)
       {
-        glnx_fd_close int repo_dfd = -1;
+        glnx_autofd int repo_dfd = -1;
         if (!glnx_opendirat (dfd, path, TRUE, &repo_dfd, error))
           return FALSE;
 
@@ -1828,7 +1826,7 @@ repo_create_at_internal (int             dfd,
         return glnx_throw_errno_prefix (error, "mkdirat");
     }
 
-  glnx_fd_close int repo_dfd = -1;
+  glnx_autofd int repo_dfd = -1;
   if (!glnx_opendirat (dfd, path, TRUE, &repo_dfd, error))
     return FALSE;
 
@@ -1919,7 +1917,7 @@ ostree_repo_create (OstreeRepo     *self,
     g_variant_builder_add (builder, "{s@v}", "collection-id",
                            g_variant_new_variant (g_variant_new_string (self->collection_id)));
 
-  glnx_fd_close int repo_dir_fd = -1;
+  glnx_autofd int repo_dir_fd = -1;
   if (!repo_create_at_internal (AT_FDCWD, repopath, mode,
                                 g_variant_builder_end (builder),
                                 &repo_dir_fd,
@@ -1963,7 +1961,7 @@ ostree_repo_create_at (int             dfd,
                        GCancellable   *cancellable,
                        GError        **error)
 {
-  glnx_fd_close int repo_dfd = -1;
+  glnx_autofd int repo_dfd = -1;
   if (!repo_create_at_internal (dfd, path, mode, options, &repo_dfd,
                                 cancellable, error))
     return NULL;
@@ -2528,14 +2526,12 @@ ostree_repo_set_cache_dir (OstreeRepo    *self,
                            GCancellable  *cancellable,
                            GError        **error)
 {
-  int fd;
-
+  glnx_autofd int fd = -1;
   if (!glnx_opendirat (dfd, path, TRUE, &fd, error))
     return FALSE;
 
-  if (self->cache_dir_fd != -1)
-    close (self->cache_dir_fd);
-  self->cache_dir_fd = fd;
+  glnx_close_fd (&self->cache_dir_fd);
+  self->cache_dir_fd = glnx_steal_fd (&fd);
 
   return TRUE;
 }
@@ -2807,16 +2803,17 @@ load_metadata_internal (OstreeRepo       *self,
                         GVariant        **out_variant,
                         GInputStream    **out_stream,
                         guint64          *out_size,
+                        OstreeRepoCommitState *out_state,
                         GCancellable     *cancellable,
                         GError          **error)
 {
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
-  struct stat stbuf;
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   g_autoptr(GInputStream) ret_stream = NULL;
   g_autoptr(GVariant) ret_variant = NULL;
 
   g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
+  g_return_val_if_fail (objtype == OSTREE_OBJECT_TYPE_COMMIT || out_state == NULL, FALSE);
 
   /* Special caching for dirmeta objects, since they're commonly referenced many
    * times.
@@ -2853,36 +2850,14 @@ load_metadata_internal (OstreeRepo       *self,
 
   if (fd != -1)
     {
+      struct stat stbuf;
       if (!glnx_fstat (fd, &stbuf, error))
         return FALSE;
-
       if (out_variant)
         {
-          /* http://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access */
-          if (stbuf.st_size > 16*1024)
-            {
-              GMappedFile *mfile;
-
-              mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
-              if (!mfile)
-                return FALSE;
-              ret_variant = g_variant_new_from_data (ostree_metadata_variant_type (objtype),
-                                                     g_mapped_file_get_contents (mfile),
-                                                     g_mapped_file_get_length (mfile),
-                                                     TRUE,
-                                                     (GDestroyNotify) g_mapped_file_unref,
-                                                     mfile);
-              g_variant_ref_sink (ret_variant);
-            }
-          else
-            {
-              g_autoptr(GBytes) data = glnx_fd_readall_bytes (fd, cancellable, error);
-              if (!data)
-                return FALSE;
-              ret_variant = g_variant_new_from_bytes (ostree_metadata_variant_type (objtype),
-                                                      data, TRUE);
-              g_variant_ref_sink (ret_variant);
-            }
+          if (!ot_variant_read_fd (fd, 0, ostree_metadata_variant_type (objtype), TRUE,
+                                   &ret_variant, error))
+            return FALSE;
 
           /* Now, let's put it in the cache */
           if (is_dirmeta_cachable)
@@ -2904,11 +2879,24 @@ load_metadata_internal (OstreeRepo       *self,
 
       if (out_size)
         *out_size = stbuf.st_size;
+
+      if (out_state)
+        {
+          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (sha256);
+          *out_state = 0;
+
+          if (!glnx_fstatat_allow_noent (self->repo_dir_fd, commitpartial_path, NULL, 0, error))
+            return FALSE;
+          if (errno == 0)
+            *out_state |= OSTREE_REPO_COMMIT_STATE_PARTIAL;
+        }
     }
   else if (self->parent_repo)
     {
-      if (!ostree_repo_load_variant (self->parent_repo, objtype, sha256, &ret_variant, error))
-        return FALSE;
+      /* Directly recurse to simplify out parameters */
+      return load_metadata_internal (self->parent_repo, objtype, sha256, error_if_not_found,
+                                     out_variant, out_stream, out_size, out_state,
+                                     cancellable, error);
     }
   else if (error_if_not_found)
     {
@@ -2952,7 +2940,7 @@ repo_load_file_archive (OstreeRepo *self,
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
   _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
 
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   if (!ot_openat_ignore_enoent (self->objects_dir_fd, loose_path_buf, &fd,
                                 error))
     return FALSE;
@@ -3008,7 +2996,7 @@ _ostree_repo_load_file_bare (OstreeRepo         *self,
     }
 
   struct stat stbuf;
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   g_autofree char *ret_symlink = NULL;
   g_autoptr(GVariant) ret_xattrs = NULL;
   char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
@@ -3082,7 +3070,7 @@ _ostree_repo_load_file_bare (OstreeRepo         *self,
               ret_symlink = g_strndup (targetbuf, target_size);
             }
           /* In the symlink case, we don't want to return the bare-user fd */
-          (void) close (glnx_steal_fd (&fd));
+          glnx_close_fd (&fd);
         }
     }
   else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
@@ -3158,7 +3146,7 @@ ostree_repo_load_file (OstreeRepo         *self,
                                    cancellable, error);
   else
     {
-      glnx_fd_close int fd = -1;
+      glnx_autofd int fd = -1;
       struct stat stbuf;
       g_autofree char *symlink_target = NULL;
       g_autoptr(GVariant) ret_xattrs = NULL;
@@ -3220,7 +3208,7 @@ ostree_repo_load_object_stream (OstreeRepo         *self,
   if (OSTREE_OBJECT_TYPE_IS_META (objtype))
     {
       if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL,
-                                   &ret_input, &size,
+                                   &ret_input, &size, NULL,
                                    cancellable, error))
         return FALSE;
     }
@@ -3516,7 +3504,7 @@ ostree_repo_load_variant_if_exists (OstreeRepo       *self,
                                     GError          **error)
 {
   return load_metadata_internal (self, objtype, sha256, FALSE,
-                                 out_variant, NULL, NULL, NULL, error);
+                                 out_variant, NULL, NULL, NULL, NULL, error);
 }
 
 /**
@@ -3538,7 +3526,7 @@ ostree_repo_load_variant (OstreeRepo       *self,
                           GError          **error)
 {
   return load_metadata_internal (self, objtype, sha256, TRUE,
-                                 out_variant, NULL, NULL, NULL, error);
+                                 out_variant, NULL, NULL, NULL, NULL, error);
 }
 
 /**
@@ -3561,31 +3549,8 @@ ostree_repo_load_commit (OstreeRepo            *self,
                          OstreeRepoCommitState *out_state,
                          GError               **error)
 {
-  if (out_variant)
-    {
-      if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, TRUE,
-                                   out_variant, NULL, NULL, NULL, error))
-        return FALSE;
-    }
-
-  if (out_state)
-    {
-      g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
-      struct stat stbuf;
-
-      *out_state = 0;
-
-      if (fstatat (self->repo_dir_fd, commitpartial_path, &stbuf, 0) == 0)
-        {
-          *out_state |= OSTREE_REPO_COMMIT_STATE_PARTIAL;
-        }
-      else if (errno != ENOENT)
-        {
-          return glnx_throw_errno_prefix (error, "fstatat(%s)", commitpartial_path);
-        }
-    }
-
-  return TRUE;
+  return load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, TRUE,
+                                 out_variant, NULL, NULL, out_state, NULL, error);
 }
 
 /**
@@ -4119,7 +4084,9 @@ ostree_repo_sign_commit (OstreeRepo     *self,
    * pass the homedir so that the signing key can be imported, allowing
    * subkey signatures to be recognised. */
   g_autoptr(GError) local_error = NULL;
-  g_autoptr(GFile) verify_keydir = g_file_new_for_path (homedir);
+  g_autoptr(GFile) verify_keydir = NULL;
+  if (homedir != NULL)
+    verify_keydir = g_file_new_for_path (homedir);
   g_autoptr(OstreeGpgVerifyResult) result
     =_ostree_repo_gpg_verify_with_metadata (self, commit_data, old_metadata,
                                             NULL, verify_keydir, NULL,
@@ -4202,15 +4169,24 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo     *self,
                                        GCancellable   *cancellable,
                                        GError        **error)
 {
-  g_autoptr(GBytes) summary_data = ot_file_mapat_bytes (self->repo_dir_fd, "summary", error);
+  glnx_autofd int fd = -1;
+  if (!glnx_openat_rdonly (self->repo_dir_fd, "summary", TRUE, &fd, error))
+    return FALSE;
+  g_autoptr(GBytes) summary_data = ot_fd_readall_or_mmap (fd, 0, error);
   if (!summary_data)
     return FALSE;
+  /* Note that fd is reused below */
+  glnx_close_fd (&fd);
 
   g_autoptr(GVariant) existing_signatures = NULL;
-  if (!ot_util_variant_map_at (self->repo_dir_fd, "summary.sig",
-                               G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING),
-                               OT_VARIANT_MAP_ALLOW_NOENT, &existing_signatures, error))
+  if (!ot_openat_ignore_enoent (self->repo_dir_fd, "summary.sig", &fd, error))
     return FALSE;
+  if (fd != -1)
+    {
+      if (!ot_variant_read_fd (fd, 0, G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING),
+                               FALSE, &existing_signatures, error))
+        return FALSE;
+    }
 
   g_autoptr(GVariant) new_metadata = NULL;
   for (guint i = 0; key_id[i]; i++)
@@ -4250,7 +4226,7 @@ find_keyring (OstreeRepo          *self,
               GCancellable        *cancellable,
               GError             **error)
 {
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   if (!ot_openat_ignore_enoent (self->repo_dir_fd, remote->keyring, &fd, error))
     return FALSE;
 
@@ -4328,7 +4304,7 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo    *self,
 
       if (keyring_data != NULL)
         {
-          _ostree_gpg_verifier_add_keyring_data (verifier, keyring_data);
+          _ostree_gpg_verifier_add_keyring_data (verifier, keyring_data, remote->keyring);
           add_global_keyring_dir = FALSE;
         }
 
@@ -4759,23 +4735,21 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
           return FALSE;
 
         g_autofree char *superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to);
-        glnx_fd_close int superblock_file_fd = -1;
+        glnx_autofd int superblock_file_fd = -1;
 
         if (!glnx_openat_rdonly (self->repo_dir_fd, superblock, TRUE, &superblock_file_fd, error))
           return FALSE;
 
-        g_autoptr(GInputStream) in_stream = g_unix_input_stream_new (superblock_file_fd, FALSE);
-        if (!in_stream)
+        g_autoptr(GBytes) superblock_content = ot_fd_readall_or_mmap (superblock_file_fd, 0, error);
+        if (!superblock_content)
           return FALSE;
+        g_auto(OtChecksum) hasher = { 0, };
+        ot_checksum_init (&hasher);
+        ot_checksum_update_bytes (&hasher, superblock_content);
+        guint8 digest[OSTREE_SHA256_DIGEST_LEN];
+        ot_checksum_get_digest (&hasher, digest, sizeof (digest));
 
-        g_autofree guchar *csum = NULL;
-        if (!ot_gio_checksum_stream (in_stream,
-                                     &csum,
-                                     cancellable,
-                                     error))
-          return FALSE;
-
-        g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], ot_gvariant_new_bytearray (csum, 32));
+        g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], ot_gvariant_new_bytearray (digest, sizeof (digest)));
       }
 
     if (delta_names->len > 0)
@@ -4960,7 +4934,6 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd,
   while (!ret_tmpdir.initialized)
     {
       struct dirent *dent;
-      glnx_fd_close int existing_tmpdir_fd = -1;
       g_autoptr(GError) local_error = NULL;
 
       if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error))
@@ -4977,7 +4950,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd,
           dent->d_type != DT_DIR)
         continue;
 
-      glnx_fd_close int target_dfd = -1;
+      glnx_autofd int target_dfd = -1;
       if (!glnx_opendirat (dfd_iter.fd, dent->d_name, FALSE,
                            &target_dfd, &local_error))
         {

src/libostree/ostree-repo.h

@@ -630,6 +630,7 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo    *r
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES: Generate size information.
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS: Canonicalize permissions for bare-user-only mode.
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_ERROR_ON_UNLABELED: Emit an error if configured SELinux policy does not provide a label
+ * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME: Delete added files/directories after commit; Since: 2017.13
  */
 typedef enum {
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_NONE = 0,
@@ -637,6 +638,7 @@ typedef enum {
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES = (1 << 1),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS = (1 << 2),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_ERROR_ON_UNLABELED = (1 << 3),
+  OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME = (1 << 4),
 } OstreeRepoCommitModifierFlags;
 
 /**
@@ -846,7 +848,7 @@ typedef enum {
 /**
  * OstreeRepoCheckoutOverwriteMode:
  * @OSTREE_REPO_CHECKOUT_OVERWRITE_NONE: No special options
- * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories
+ * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories (unless whiteouts are enabled, then directories are replaced)
  * @OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES: Only add new files/directories
  * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL: Like UNION_FILES, but error if files are not identical (requires hardlink checkouts)
  */
@@ -909,6 +911,9 @@ OstreeRepoDevInoCache * ostree_repo_devino_cache_ref (OstreeRepoDevInoCache *cac
 _OSTREE_PUBLIC
 void ostree_repo_devino_cache_unref (OstreeRepoDevInoCache *cache);
 
+_OSTREE_PUBLIC
+void ostree_repo_checkout_at_options_set_devino (OstreeRepoCheckoutAtOptions *opts, OstreeRepoDevInoCache *cache);
+
 _OSTREE_PUBLIC
 gboolean ostree_repo_checkout_at (OstreeRepo                         *self,
                                   OstreeRepoCheckoutAtOptions        *options,

src/libostree/ostree-sepolicy.c

@@ -203,7 +203,7 @@ get_policy_checksum (char        **out_csum,
   g_autofree char *best_policy = NULL;
   int best_version = 0;
 
-  glnx_fd_close int bindir_dfd = -1;
+  glnx_autofd int bindir_dfd = -1;
   if (!glnx_opendirat (AT_FDCWD, bindir_path, TRUE, &bindir_dfd, error))
     return FALSE;
 

src/libostree/ostree-sysroot-cleanup.c

@@ -282,7 +282,7 @@ cleanup_old_deployments (OstreeSysroot       *self,
       if (!g_hash_table_lookup (active_deployment_dirs, deployment_path))
         {
           struct stat stbuf;
-          glnx_fd_close int deployment_fd = -1;
+          glnx_autofd int deployment_fd = -1;
 
           if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE,
                                &deployment_fd, error))

src/libostree/ostree-sysroot-deploy.c

@@ -118,6 +118,7 @@ hardlink_or_copy_at (int         src_dfd,
   return TRUE;
 }
 
+/* Copy ownership, mode, and xattrs from source directory to destination */
 static gboolean
 dirfd_copy_attributes_and_xattrs (int            src_parent_dfd,
                                   const char    *src_name,
@@ -163,7 +164,7 @@ copy_dir_recurse (int              src_parent_dfd,
                   GError         **error)
 {
   g_auto(GLnxDirFdIterator) src_dfd_iter = { 0, };
-  glnx_fd_close int dest_dfd = -1;
+  glnx_autofd int dest_dfd = -1;
   struct dirent *dent;
 
   if (!glnx_dirfd_iterator_init_at (src_parent_dfd, name, TRUE, &src_dfd_iter, error))
@@ -212,6 +213,9 @@ copy_dir_recurse (int              src_parent_dfd,
   return TRUE;
 }
 
+/* If a chain of directories is added, this function will ensure
+ * they're created.
+ */
 static gboolean
 ensure_directory_from_template (int                 orig_etc_fd,
                                 int                 modified_etc_fd,
@@ -222,8 +226,8 @@ ensure_directory_from_template (int                 orig_etc_fd,
                                 GCancellable       *cancellable,
                                 GError            **error)
 {
-  glnx_fd_close int src_dfd = -1;
-  glnx_fd_close int target_dfd = -1;
+  glnx_autofd int src_dfd = -1;
+  glnx_autofd int target_dfd = -1;
 
   g_assert (path != NULL);
   g_assert (*path != '/' && *path != '\0');
@@ -274,12 +278,9 @@ ensure_directory_from_template (int                 orig_etc_fd,
   return TRUE;
 }
 
-/**
- * copy_modified_config_file:
- *
- * Copy @file from @modified_etc to @new_etc, overwriting any existing
- * file there.  The @file may refer to a regular file, a symbolic
- * link, or a directory.  Directories will be copied recursively.
+/* Copy (relative) @path from @modified_etc_fd to @new_etc_fd, overwriting any
+ * existing file there. The @path may refer to a regular file, a symbolic link,
+ * or a directory. Directories will be copied recursively.
  */
 static gboolean
 copy_modified_config_file (int                 orig_etc_fd,
@@ -296,7 +297,7 @@ copy_modified_config_file (int                 orig_etc_fd,
   if (!glnx_fstatat (modified_etc_fd, path, &modified_stbuf, AT_SYMLINK_NOFOLLOW, error))
     return glnx_prefix_error (error, "Reading modified config file");
 
-  glnx_fd_close int dest_parent_dfd = -1;
+  glnx_autofd int dest_parent_dfd = -1;
   if (strchr (path, '/') != NULL)
     {
       g_autofree char *parent = g_path_get_dirname (path);
@@ -392,7 +393,7 @@ merge_configuration_from (OstreeSysroot    *sysroot,
                           GCancellable     *cancellable,
                           GError          **error)
 {
-  glnx_fd_close int owned_merge_deployment_dfd = -1;
+  glnx_autofd int owned_merge_deployment_dfd = -1;
   const OstreeSysrootDebugFlags flags = sysroot->debug_flags;
 
   g_assert (merge_deployment != NULL && new_deployment != NULL);
@@ -442,13 +443,13 @@ merge_configuration_from (OstreeSysroot    *sysroot,
     _ostree_sysroot_emit_journal_msg (sysroot, msg);
   }
 
-  glnx_fd_close int orig_etc_fd = -1;
+  glnx_autofd int orig_etc_fd = -1;
   if (!glnx_opendirat (merge_deployment_dfd, "usr/etc", TRUE, &orig_etc_fd, error))
     return FALSE;
-  glnx_fd_close int modified_etc_fd = -1;
+  glnx_autofd int modified_etc_fd = -1;
   if (!glnx_opendirat (merge_deployment_dfd, "etc", TRUE, &modified_etc_fd, error))
     return FALSE;
-  glnx_fd_close int new_etc_fd = -1;
+  glnx_autofd int new_etc_fd = -1;
   if (!glnx_opendirat (new_deployment_dfd, "etc", TRUE, &new_etc_fd, error))
     return FALSE;
 
@@ -490,11 +491,9 @@ merge_configuration_from (OstreeSysroot    *sysroot,
   return TRUE;
 }
 
-/**
- * checkout_deployment_tree:
- *
- * Look up @revision in the repository, and check it out in
+/* Look up @revision in the repository, and check it out in
  * /ostree/deploy/OS/deploy/${treecsum}.${deployserial}.
+ * A dfd for the result is returned in @out_deployment_dfd.
  */
 static gboolean
 checkout_deployment_tree (OstreeSysroot     *sysroot,
@@ -509,7 +508,7 @@ checkout_deployment_tree (OstreeSysroot     *sysroot,
   const char *csum = ostree_deployment_get_csum (deployment);
   g_autofree char *checkout_target_name = NULL;
   g_autofree char *osdeploy_path = NULL;
-  glnx_fd_close int osdeploy_dfd = -1;
+  glnx_autofd int osdeploy_dfd = -1;
   int ret_fd;
 
   osdeploy_path = g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL);
@@ -676,6 +675,9 @@ selinux_relabel_dir (OstreeSysroot                 *sysroot,
   return ret;
 }
 
+/* Handles SELinux labeling for /var; this is slated to be deleted.  See
+ * https://github.com/ostreedev/ostree/pull/872
+ */
 static gboolean
 selinux_relabel_var_if_needed (OstreeSysroot                 *sysroot,
                                OstreeSePolicy                *sepolicy,
@@ -730,6 +732,11 @@ selinux_relabel_var_if_needed (OstreeSysroot                 *sysroot,
   return TRUE;
 }
 
+/* OSTree implements a "3 way" merge model for /etc. For a bit more information
+ * on this, see the manual. This function uses the configuration for
+ * @previous_deployment, and writes the merged configuration into @deployment's
+ * /etc.  If available, we also load the SELinux policy from the new root.
+ */
 static gboolean
 merge_configuration (OstreeSysroot         *sysroot,
                      OstreeRepo            *repo,
@@ -820,6 +827,7 @@ merge_configuration (OstreeSysroot         *sysroot,
   return TRUE;
 }
 
+/* Write the origin file for a deployment. */
 static gboolean
 write_origin_file_internal (OstreeSysroot         *sysroot,
                             OstreeDeployment      *deployment,
@@ -891,8 +899,7 @@ typedef struct {
 static void
 _ostree_kernel_layout_free (OstreeKernelLayout *layout)
 {
-  if (layout->boot_dfd != -1)
-    (void) close (layout->boot_dfd);
+  glnx_close_fd (&layout->boot_dfd);
   g_free (layout->kernel_srcpath);
   g_free (layout->kernel_namever);
   g_free (layout->initramfs_srcpath);
@@ -981,16 +988,17 @@ get_kernel_from_tree_usrlib_modules (int                  deployment_dfd,
     }
 
   /* We found a module directory, compute the checksum */
-  g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256);
-  glnx_fd_close int fd = -1;
+  g_auto(OtChecksum) checksum = { 0, };
+  ot_checksum_init (&checksum);
+  glnx_autofd int fd = -1;
   /* Checksum the kernel */
   if (!glnx_openat_rdonly (ret_layout->boot_dfd, "vmlinuz", TRUE, &fd, error))
     return FALSE;
   g_autoptr(GInputStream) in = g_unix_input_stream_new (fd, FALSE);
-  if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+  if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error))
     return FALSE;
   g_clear_object (&in);
-  (void) close (fd); fd = -1;
+  glnx_close_fd (&fd);
 
   /* Look for an initramfs, but it's optional; since there wasn't any precedent
    * for this, let's be a bit conservative and support both `initramfs.img` and
@@ -1014,11 +1022,13 @@ get_kernel_from_tree_usrlib_modules (int                  deployment_dfd,
       ret_layout->initramfs_srcpath = g_strdup (initramfs_path);
       ret_layout->initramfs_namever = g_strdup_printf ("initramfs-%s.img", kver);
       in = g_unix_input_stream_new (fd, FALSE);
-      if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+      if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error))
         return FALSE;
     }
 
-  ret_layout->bootcsum = g_strdup (g_checksum_get_string (checksum));
+  char hexdigest[OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&checksum, hexdigest, sizeof (hexdigest));
+  ret_layout->bootcsum = g_strdup (hexdigest);
 
   *out_layout = g_steal_pointer (&ret_layout);
   return TRUE;
@@ -1227,8 +1237,8 @@ fsfreeze_thaw_cycle (OstreeSysroot *self,
   int sockpair[2];
   if (socketpair (AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sockpair) < 0)
     return glnx_throw_errno_prefix (error, "socketpair");
-  glnx_fd_close int sock_parent = sockpair[0];
-  glnx_fd_close int sock_watchdog = sockpair[1];
+  glnx_autofd int sock_parent = sockpair[0];
+  glnx_autofd int sock_watchdog = sockpair[1];
 
   pid_t pid = fork ();
   if (pid < 0)
@@ -1238,7 +1248,7 @@ fsfreeze_thaw_cycle (OstreeSysroot *self,
   char c = '!';
   if (pid == 0) /* Child watchdog/unfreezer process. */
     {
-      (void) close (glnx_steal_fd (&sock_parent));
+      glnx_close_fd (&sock_parent);
       /* Daemonize, and mask SIGINT/SIGTERM, so we're likely to survive e.g.
        * someone doing a `systemctl restart rpm-ostreed` or a Ctrl-C of
        * `ostree admin upgrade`.  We don't daemonize though if testing so
@@ -1291,11 +1301,16 @@ fsfreeze_thaw_cycle (OstreeSysroot *self,
         }
       if (debug_fifreeze)
         g_printerr ("fifreeze watchdog was run\n");
-      exit (EXIT_SUCCESS);
+      /* We use _exit() rather than exit() to avoid tripping over any shared
+       * libraries in process that aren't fork() safe; for example gjs/spidermonkey:
+       * https://github.com/ostreedev/ostree/issues/1262
+       * This doesn't help for the err()/errx() calls above, but eh...
+       */
+      _exit (EXIT_SUCCESS);
     }
   else /* Parent process. */
     {
-      (void) close (glnx_steal_fd (&sock_watchdog));
+      glnx_close_fd (&sock_watchdog);
       /* Wait for the watchdog to say it's set up; mainly that it's
        * masked SIGTERM successfully.
        */
@@ -1321,11 +1336,15 @@ fsfreeze_thaw_cycle (OstreeSysroot *self,
       /* Do a freeze/thaw cycle; TODO add a FIFREEZETHAW ioctl */
       if (ioctl (rootfs_dfd, FIFREEZE, 0) != 0)
         {
-          /* Not supported, or we're running in the unit tests (as non-root)?
+          /* Not supported, we're running in the unit tests (as non-root), or
+           * the filesystem is already frozen (EBUSY).
            * OK, let's just do a syncfs.
            */
-          if (G_IN_SET (errno, EOPNOTSUPP, EPERM))
+          if (G_IN_SET (errno, EOPNOTSUPP, EPERM, EBUSY))
             {
+              /* Warn if the filesystem was already frozen */
+              if (errno == EBUSY)
+                g_debug ("Filesystem already frozen, falling back to syncfs");
               if (TEMP_FAILURE_RETRY (syncfs (rootfs_dfd)) != 0)
                 return glnx_throw_errno_prefix (error, "syncfs");
               /* Write the completion, and return */
@@ -1338,7 +1357,13 @@ fsfreeze_thaw_cycle (OstreeSysroot *self,
         }
       /* And finally thaw, then signal our completion to the watchdog */
       if (TEMP_FAILURE_RETRY (ioctl (rootfs_dfd, FITHAW, 0)) != 0)
-        return glnx_throw_errno_prefix (error, "ioctl(FITHAW)");
+        {
+          /* Warn but don't error if the filesystem was already thawed */
+          if (errno == EINVAL)
+            g_debug ("Filesystem already thawed");
+          else
+            return glnx_throw_errno_prefix (error, "ioctl(FITHAW)");
+        }
       if (write (sock_parent, &c, sizeof (c)) != sizeof (c))
         return glnx_throw_errno_prefix (error, "write(watchdog FITHAW complete)");
     }
@@ -1369,7 +1394,7 @@ full_system_sync (OstreeSysroot     *self,
   out_stats->root_syncfs_msec = (end_msec - start_msec);
 
   start_msec = g_get_monotonic_time () / 1000;
-  glnx_fd_close int boot_dfd = -1;
+  glnx_autofd int boot_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, "boot", TRUE, &boot_dfd, error))
     return FALSE;
   if (!fsfreeze_thaw_cycle (self, boot_dfd, cancellable, error))
@@ -1391,6 +1416,12 @@ full_system_sync (OstreeSysroot     *self,
   return TRUE;
 }
 
+/* Write out the "bootlinks", which are symlinks pointing to deployments.
+ * We might be generating a new bootversion (i.e. updating the bootloader config),
+ * or we might just be generating a "sub-bootversion".
+ *
+ * These new links are made active by swap_bootlinks().
+ */
 static gboolean
 create_new_bootlinks (OstreeSysroot *self,
                       int            bootversion,
@@ -1398,7 +1429,7 @@ create_new_bootlinks (OstreeSysroot *self,
                        GCancellable  *cancellable,
                       GError       **error)
 {
-  glnx_fd_close int ostree_dfd = -1;
+  glnx_autofd int ostree_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, "ostree", TRUE, &ostree_dfd, error))
     return FALSE;
 
@@ -1423,7 +1454,7 @@ create_new_bootlinks (OstreeSysroot *self,
   if (!glnx_shutil_mkdir_p_at (ostree_dfd, ostree_subbootdir_name, 0755, cancellable, error))
     return FALSE;
 
-  glnx_fd_close int ostree_subbootdir_dfd = -1;
+  glnx_autofd int ostree_subbootdir_dfd = -1;
   if (!glnx_opendirat (ostree_dfd, ostree_subbootdir_name, FALSE, &ostree_subbootdir_dfd, error))
     return FALSE;
 
@@ -1451,6 +1482,8 @@ create_new_bootlinks (OstreeSysroot *self,
   return TRUE;
 }
 
+/* Rename into place symlinks created via create_new_bootlinks().
+ */
 static gboolean
 swap_bootlinks (OstreeSysroot *self,
                 int            bootversion,
@@ -1458,7 +1491,7 @@ swap_bootlinks (OstreeSysroot *self,
                 GCancellable  *cancellable,
                 GError       **error)
 {
-  glnx_fd_close int ostree_dfd = -1;
+  glnx_autofd int ostree_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, "ostree", TRUE, &ostree_dfd, error))
     return FALSE;
 
@@ -1512,10 +1545,9 @@ parse_os_release (const char *contents,
   return ret;
 }
 
-/*
- * install_deployment_kernel:
- *
- * Write out an entry in /boot/loader/entries for @deployment.
+/* Given @deployment, prepare it to be booted; basically copying its
+ * kernel/initramfs into /boot/ostree (if needed) and writing out an entry in
+ * /boot/loader/entries.
  */
 static gboolean
 install_deployment_kernel (OstreeSysroot   *sysroot,
@@ -1530,7 +1562,7 @@ install_deployment_kernel (OstreeSysroot   *sysroot,
 {
   OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (deployment);
   g_autofree char *deployment_dirpath = ostree_sysroot_get_deployment_dirpath (sysroot, deployment);
-  glnx_fd_close int deployment_dfd = -1;
+  glnx_autofd int deployment_dfd = -1;
   if (!glnx_opendirat (sysroot->sysroot_fd, deployment_dirpath, FALSE,
                        &deployment_dfd, error))
     return FALSE;
@@ -1541,7 +1573,7 @@ install_deployment_kernel (OstreeSysroot   *sysroot,
                              cancellable, error))
     return FALSE;
 
-  glnx_fd_close int boot_dfd = -1;
+  glnx_autofd int boot_dfd = -1;
   if (!glnx_opendirat (sysroot->sysroot_fd, "boot", TRUE, &boot_dfd, error))
     return FALSE;
 
@@ -1555,7 +1587,7 @@ install_deployment_kernel (OstreeSysroot   *sysroot,
   if (!glnx_shutil_mkdir_p_at (boot_dfd, bootcsumdir, 0775, cancellable, error))
     return FALSE;
 
-  glnx_fd_close int bootcsum_dfd = -1;
+  glnx_autofd int bootcsum_dfd = -1;
   if (!glnx_opendirat (boot_dfd, bootcsumdir, TRUE, &bootcsum_dfd, error))
     return FALSE;
 
@@ -1680,13 +1712,13 @@ install_deployment_kernel (OstreeSysroot   *sysroot,
   g_autofree char *ostree_kernel_arg = g_strdup_printf ("ostree=/ostree/boot.%d/%s/%s/%d",
                                        new_bootversion, osname, bootcsum,
                                        ostree_deployment_get_bootserial (deployment));
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = _ostree_kernel_args_from_string (val);
+  g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_from_string (val);
   _ostree_kernel_args_replace_take (kargs, ostree_kernel_arg);
   ostree_kernel_arg = NULL;
   g_autofree char *options_key = _ostree_kernel_args_to_string (kargs);
   ostree_bootconfig_parser_set (bootconfig, "options", options_key);
 
-  glnx_fd_close int bootconf_dfd = -1;
+  glnx_autofd int bootconf_dfd = -1;
   if (!glnx_opendirat (boot_dfd, bootconfdir, TRUE, &bootconf_dfd, error))
     return FALSE;
 
@@ -1698,6 +1730,10 @@ install_deployment_kernel (OstreeSysroot   *sysroot,
   return TRUE;
 }
 
+/* We generate the symlink on disk, then potentially do a syncfs() to ensure
+ * that it (and everything else we wrote) has hit disk. Only after that do we
+ * rename it into place.
+ */
 static gboolean
 prepare_new_bootloader_link (OstreeSysroot  *sysroot,
                              int             current_bootversion,
@@ -1719,6 +1755,7 @@ prepare_new_bootloader_link (OstreeSysroot  *sysroot,
   return TRUE;
 }
 
+/* Update the /boot/loader symlink to point to /boot/loader.$new_bootversion */
 static gboolean
 swap_bootloader (OstreeSysroot  *sysroot,
                  int             current_bootversion,
@@ -1726,7 +1763,7 @@ swap_bootloader (OstreeSysroot  *sysroot,
                  GCancellable   *cancellable,
                  GError        **error)
 {
-  glnx_fd_close int boot_dfd = -1;
+  glnx_autofd int boot_dfd = -1;
 
   g_assert ((current_bootversion == 0 && new_bootversion == 1) ||
             (current_bootversion == 1 && new_bootversion == 0));
@@ -1778,6 +1815,15 @@ assign_bootserials (GPtrArray   *deployments)
   return ret;
 }
 
+/* OSTree implements a special optimization where we want to avoid touching
+ * the bootloader configuration if the kernel layout hasn't changed.  This is
+ * handled by the ostree= kernel argument referring to a "bootlink".  But
+ * we *do* need to update the bootloader configuration if the kernel arguments
+ * change.
+ *
+ * Hence, this function determines if @a and @b are fully compatible from a
+ * bootloader perspective.
+ */
 static gboolean
 deployment_bootconfigs_equal (OstreeDeployment *a,
                               OstreeDeployment *b)
@@ -1793,8 +1839,8 @@ deployment_bootconfigs_equal (OstreeDeployment *a,
     OstreeBootconfigParser *b_bootconfig = ostree_deployment_get_bootconfig (b);
     const char *a_boot_options = ostree_bootconfig_parser_get (a_bootconfig, "options");
     const char *b_boot_options = ostree_bootconfig_parser_get (b_bootconfig, "options");
-    __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *a_kargs = NULL;
-    __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *b_kargs = NULL;
+    g_autoptr(OstreeKernelArgs) a_kargs = NULL;
+    g_autoptr(OstreeKernelArgs) b_kargs = NULL;
     g_autofree char *a_boot_options_without_ostree = NULL;
     g_autofree char *b_boot_options_without_ostree = NULL;
 
@@ -1841,6 +1887,11 @@ cleanup_legacy_current_symlinks (OstreeSysroot         *self,
   return TRUE;
 }
 
+/* Detect whether or not @path refers to a read-only mountpoint. This is
+ * currently just used to handle a potentially read-only /boot by transiently
+ * remounting it read-write. In the future we might also do this for e.g.
+ * /sysroot.
+ */
 static gboolean
 is_ro_mount (const char *path)
 {
@@ -2207,7 +2258,7 @@ allocate_deployserial (OstreeSysroot           *self,
   g_autoptr(GPtrArray) tmp_current_deployments =
     g_ptr_array_new_with_free_func (g_object_unref);
 
-  glnx_fd_close int deploy_dfd = -1;
+  glnx_autofd int deploy_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, "ostree/deploy", TRUE, &deploy_dfd, error))
     return FALSE;
 
@@ -2262,7 +2313,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot     *self,
     osname = ostree_deployment_get_osname (self->booted_deployment);
 
   const char *osdeploypath = glnx_strjoina ("ostree/deploy/", osname);
-  glnx_fd_close int os_deploy_dfd = -1;
+  glnx_autofd int os_deploy_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, osdeploypath, TRUE, &os_deploy_dfd, error))
     return FALSE;
 
@@ -2283,7 +2334,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot     *self,
   ostree_deployment_set_origin (new_deployment, origin);
 
   /* Check out the userspace tree onto the filesystem */
-  glnx_fd_close int deployment_dfd = -1;
+  glnx_autofd int deployment_dfd = -1;
   if (!checkout_deployment_tree (self, repo, new_deployment, &deployment_dfd,
                                  cancellable, error))
     {
@@ -2349,7 +2400,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot     *self,
    */
   if (override_kernel_argv)
     {
-      __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = NULL;
+      g_autoptr(OstreeKernelArgs) kargs = NULL;
       g_autofree char *new_options = NULL;
 
       kargs = _ostree_kernel_args_new ();
@@ -2380,22 +2431,16 @@ ostree_sysroot_deployment_set_kargs (OstreeSysroot     *self,
                                      GCancellable      *cancellable,
                                      GError           **error)
 {
-  guint i;
-  g_autoptr(GPtrArray) new_deployments = g_ptr_array_new_with_free_func (g_object_unref);
-  g_autoptr(OstreeDeployment) new_deployment = NULL;
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = NULL;
-  g_autofree char *new_options = NULL;
-  OstreeBootconfigParser *new_bootconfig;
+  g_autoptr(OstreeDeployment) new_deployment = ostree_deployment_clone (deployment);
+  OstreeBootconfigParser *new_bootconfig = ostree_deployment_get_bootconfig (new_deployment);
 
-  new_deployment = ostree_deployment_clone (deployment);
-  new_bootconfig = ostree_deployment_get_bootconfig (new_deployment);
-
-  kargs = _ostree_kernel_args_new ();
+  g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_new ();
   _ostree_kernel_args_append_argv (kargs, new_kargs);
-  new_options = _ostree_kernel_args_to_string (kargs);
+  g_autofree char *new_options = _ostree_kernel_args_to_string (kargs);
   ostree_bootconfig_parser_set (new_bootconfig, "options", new_options);
 
-  for (i = 0; i < self->deployments->len; i++)
+  g_autoptr(GPtrArray) new_deployments = g_ptr_array_new_with_free_func (g_object_unref);
+  for (guint i = 0; i < self->deployments->len; i++)
     {
       OstreeDeployment *cur = self->deployments->pdata[i];
       if (cur == deployment)
@@ -2435,7 +2480,7 @@ ostree_sysroot_deployment_set_mutable (OstreeSysroot     *self,
     return FALSE;
 
   g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment);
-  glnx_fd_close int fd = -1;
+  glnx_autofd int fd = -1;
   if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE, &fd, error))
     return FALSE;
 

src/libostree/ostree-sysroot.c

@@ -22,6 +22,7 @@
 #include "otutil.h"
 #include <sys/file.h>
 #include <sys/mount.h>
+#include <err.h>
 #include <sys/wait.h>
 
 #include "ostree.h"
@@ -201,7 +202,12 @@ ostree_sysroot_init (OstreeSysroot *self)
 
 /**
  * ostree_sysroot_new:
- * @path: (allow-none): Path to a system root directory, or %NULL
+ * @path: (allow-none): Path to a system root directory, or %NULL to use the
+ *   current visible root file system
+ *
+ * Create a new #OstreeSysroot object for the sysroot at @path. If @path is %NULL,
+ * the current visible root file system is used, equivalent to
+ * ostree_sysroot_new_default().
  *
  * Returns: (transfer full): An accessor object for an system root located at @path
  */
@@ -291,11 +297,7 @@ _ostree_sysroot_bump_mtime (OstreeSysroot *self,
 void
 ostree_sysroot_unload (OstreeSysroot  *self)
 {
-  if (self->sysroot_fd != -1)
-    {
-      (void) close (self->sysroot_fd);
-      self->sysroot_fd = -1;
-    }
+  glnx_close_fd (&self->sysroot_fd);
 }
 
 /**
@@ -633,7 +635,7 @@ parse_deployment (OstreeSysroot       *self,
                                                &treecsum, &deployserial, error))
     return FALSE;
 
-  glnx_fd_close int deployment_dfd = -1;
+  glnx_autofd int deployment_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, relative_boot_link, TRUE,
                        &deployment_dfd, error))
     return FALSE;
@@ -981,11 +983,14 @@ ostree_sysroot_get_deployment_origin_path (GFile   *deployment_path)
 /**
  * ostree_sysroot_get_repo:
  * @self: Sysroot
- * @out_repo: (out): Repository in sysroot @self
+ * @out_repo: (out) (transfer full) (optional): Repository in sysroot @self
  * @cancellable: Cancellable
  * @error: Error
  *
- * Retrieve the OSTree repository in sysroot @self.
+ * Retrieve the OSTree repository in sysroot @self. The repo is guaranteed to be open
+ * (see ostree_repo_open()).
+ *
+ * Returns: %TRUE on success, %FALSE otherwise
  */
 gboolean
 ostree_sysroot_get_repo (OstreeSysroot         *self,
@@ -1125,7 +1130,7 @@ find_booted_deployment (OstreeSysroot       *self,
   if (root_stbuf.st_dev == self_stbuf.st_dev &&
       root_stbuf.st_ino == self_stbuf.st_ino)
     {
-      __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kernel_args = NULL;
+      g_autoptr(OstreeKernelArgs) kernel_args = NULL;
       if (!parse_kernel_commandline (&kernel_args, cancellable, error))
         return FALSE;
 
@@ -1433,7 +1438,7 @@ ostree_sysroot_init_osname (OstreeSysroot       *self,
   if (mkdirat (self->sysroot_fd, deploydir, 0777) < 0)
     return glnx_throw_errno_prefix (error, "Creating %s", deploydir);
 
-  glnx_fd_close int dfd = -1;
+  glnx_autofd int dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, deploydir, TRUE, &dfd, error))
     return FALSE;
 
@@ -1615,7 +1620,7 @@ clone_deployment (OstreeSysroot  *sysroot,
   /* Copy the bootloader config options */
   OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (merge_deployment);
   g_auto(GStrv) previous_args = g_strsplit (ostree_bootconfig_parser_get (bootconfig, "options"), " ", -1);
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = _ostree_kernel_args_new ();
+  g_autoptr(OstreeKernelArgs) kargs = _ostree_kernel_args_new ();
   _ostree_kernel_args_append_argv (kargs, previous_args);
 
   /* Deploy the copy */
@@ -1689,7 +1694,7 @@ ostree_sysroot_deployment_unlock (OstreeSysroot     *self,
     return FALSE;
 
   g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment);
-  glnx_fd_close int deployment_dfd = -1;
+  glnx_autofd int deployment_dfd = -1;
   if (!glnx_opendirat (self->sysroot_fd, deployment_path, TRUE, &deployment_dfd, error))
     return FALSE;
 
@@ -1698,6 +1703,7 @@ ostree_sysroot_deployment_unlock (OstreeSysroot     *self,
     return FALSE;
 
   const char *ovl_options = NULL;
+  static const char hotfix_ovl_options[] = "lowerdir=usr,upperdir=.usr-ovl-upper,workdir=.usr-ovl-work";
   switch (unlocked_state)
     {
     case OSTREE_DEPLOYMENT_UNLOCKED_NONE:
@@ -1705,7 +1711,6 @@ ostree_sysroot_deployment_unlock (OstreeSysroot     *self,
       break;
     case OSTREE_DEPLOYMENT_UNLOCKED_HOTFIX:
       {
-        const char hotfix_ovl_options[] = "lowerdir=usr,upperdir=.usr-ovl-upper,workdir=.usr-ovl-work";
         /* Create the overlayfs directories in the deployment root
          * directly for hotfixes.  The ostree-prepare-root.c helper
          * is also set up to detect and mount these.
@@ -1763,11 +1768,15 @@ ostree_sysroot_deployment_unlock (OstreeSysroot     *self,
       return glnx_throw_errno_prefix (error, "fork");
     else if (mount_child == 0)
       {
-        /* Child process.  Do NOT use any GLib API here. */
+        /* Child process. Do NOT use any GLib API here; it's not generally fork() safe.
+         *
+         * TODO: report errors across a pipe (or use the journal?) rather than
+         * spewing to stderr.
+         */
         if (fchdir (deployment_dfd) < 0)
-          exit (EXIT_FAILURE);
+          err (1, "fchdir");
         if (mount ("overlay", "/usr", "overlay", 0, ovl_options) < 0)
-          exit (EXIT_FAILURE);
+          err (1, "mount");
         exit (EXIT_SUCCESS);
       }
     else
@@ -1778,7 +1787,7 @@ ostree_sysroot_deployment_unlock (OstreeSysroot     *self,
         if (TEMP_FAILURE_RETRY (waitpid (mount_child, &estatus, 0)) < 0)
           return glnx_throw_errno_prefix (error, "waitpid() on mount helper");
         if (!g_spawn_check_exit_status (estatus, error))
-          return glnx_throw_errno_prefix (error, "overlayfs mount helper");
+          return glnx_prefix_error (error, "Failed overlayfs mount");
       }
   }
 

src/libostree/ostree-version.h

@@ -43,7 +43,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_RELEASE_VERSION (12)
+#define OSTREE_RELEASE_VERSION (13)
 
 /**
  * OSTREE_VERSION
@@ -52,7 +52,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION (2017.12)
+#define OSTREE_VERSION (2017.13)
 
 /**
  * OSTREE_VERSION_S:
@@ -62,7 +62,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION_S "2017.12"
+#define OSTREE_VERSION_S "2017.13"
 
 #define OSTREE_ENCODE_VERSION(year,release) \
         ((year) << 16 | (release))

src/libostree/ostree.h

@@ -40,6 +40,7 @@
 #include <ostree-repo-finder-avahi.h>
 #include <ostree-repo-finder-config.h>
 #include <ostree-repo-finder-mount.h>
+#include <ostree-repo-finder-override.h>
 #endif /* OSTREE_ENABLE_EXPERIMENTAL_API */
 
 #include <ostree-autocleanups.h>

src/libotutil/ot-checksum-instream.c

@@ -22,25 +22,10 @@
 #include "ot-checksum-instream.h"
 #include "ot-checksum-utils.h"
 
-#if defined(HAVE_OPENSSL)
-#include <openssl/evp.h>
-#elif defined(HAVE_GNUTLS)
-#include <gnutls/gnutls.h>
-#include <gnutls/crypto.h>
-#endif
-
 G_DEFINE_TYPE (OtChecksumInstream, ot_checksum_instream, G_TYPE_FILTER_INPUT_STREAM)
 
 struct _OtChecksumInstreamPrivate {
-#if defined(HAVE_OPENSSL)
-  EVP_MD_CTX *checksum;
-#elif defined(HAVE_GNUTLS)
-  gnutls_digest_algorithm_t checksum_type;
-  gnutls_hash_hd_t checksum;
-#else
-  GChecksumType checksum_type;
-  GChecksum *checksum;
-#endif
+  OtChecksum checksum;
 };
 
 static gssize   ot_checksum_instream_read         (GInputStream         *stream,
@@ -54,13 +39,7 @@ ot_checksum_instream_finalize (GObject *object)
 {
   OtChecksumInstream *self = (OtChecksumInstream*)object;
 
-#if defined(HAVE_OPENSSL)
-  EVP_MD_CTX_destroy (self->priv->checksum);
-#elif defined(HAVE_GNUTLS)
-  gnutls_hash_deinit (self->priv->checksum, NULL);
-#else
-  g_checksum_free (self->priv->checksum);
-#endif
+  ot_checksum_clear (&self->priv->checksum);
 
   G_OBJECT_CLASS (ot_checksum_instream_parent_class)->finalize (object);
 }
@@ -83,33 +62,6 @@ ot_checksum_instream_init (OtChecksumInstream *self)
   self->priv = G_TYPE_INSTANCE_GET_PRIVATE (self, OT_TYPE_CHECKSUM_INSTREAM, OtChecksumInstreamPrivate);
 }
 
-#if defined(HAVE_OPENSSL)
-static const EVP_MD *
-gchecksum_type_to_openssl (GChecksumType checksum_type)
-{
-  switch (checksum_type)
-    {
-    case G_CHECKSUM_SHA256:
-      return EVP_sha256 ();
-    default:
-      /* If there's something else, fill in here */
-      g_assert_not_reached ();
-    }
-}
-#elif defined(HAVE_GNUTLS)
-static gnutls_digest_algorithm_t
-gchecksum_type_to_gnutls (GChecksumType checksum_type)
-{
-  switch (checksum_type)
-    {
-    case G_CHECKSUM_SHA256:
-      return GNUTLS_DIG_SHA256;
-    default:
-      g_assert_not_reached ();
-    }
-}
-#endif
-
 OtChecksumInstream *
 ot_checksum_instream_new (GInputStream    *base,
                           GChecksumType    checksum_type)
@@ -124,18 +76,7 @@ ot_checksum_instream_new (GInputStream    *base,
 
   /* For now */
   g_assert (checksum_type == G_CHECKSUM_SHA256);
-
-#if defined(HAVE_OPENSSL)
-  stream->priv->checksum = EVP_MD_CTX_create ();
-  g_assert (stream->priv->checksum);
-  g_assert (EVP_DigestInit_ex (stream->priv->checksum, gchecksum_type_to_openssl (checksum_type), NULL));
-#elif defined(HAVE_GNUTLS)
-  stream->priv->checksum_type = gchecksum_type_to_gnutls (checksum_type);
-  g_assert (!gnutls_hash_init (&stream->priv->checksum, stream->priv->checksum_type));
-#else
-  stream->priv->checksum = g_checksum_new (checksum_type);
-  stream->priv->checksum_type = checksum_type;
-#endif
+  ot_checksum_init (&stream->priv->checksum);
 
   return (OtChecksumInstream*) (stream);
 }
@@ -157,78 +98,15 @@ ot_checksum_instream_read (GInputStream  *stream,
                              cancellable,
                              error);
   if (res > 0)
-    {
-#if defined(HAVE_OPENSSL)
-      g_assert (EVP_DigestUpdate (self->priv->checksum, buffer, res));
-#elif defined(HAVE_GNUTLS)
-      g_assert (!gnutls_hash (self->priv->checksum, buffer, res));
-#else
-      g_checksum_update (self->priv->checksum, buffer, res);
-#endif
-    }
+    ot_checksum_update (&self->priv->checksum, buffer, res);
 
   return res;
 }
 
-void
-ot_checksum_instream_get_digest (OtChecksumInstream *stream,
-                                 guint8          *buffer,
-                                 gsize           *digest_len)
-{
-#if defined(HAVE_OPENSSL)
-  unsigned len;
-  EVP_DigestFinal_ex (stream->priv->checksum, buffer, &len);
-  if (digest_len)
-    *digest_len = len;
-#elif defined(HAVE_GNUTLS)
-  gnutls_hash_output (stream->priv->checksum, buffer);
-  if (digest_len)
-    *digest_len = gnutls_hash_get_len (stream->priv->checksum_type);
-#else
-  g_checksum_get_digest (stream->priv->checksum, buffer, digest_len);
-#endif
-}
-
-guint8*
-ot_checksum_instream_dup_digest (OtChecksumInstream *stream,
-                                 gsize              *ret_len)
-{
-#if defined(HAVE_OPENSSL)
-  guint len;
-  guchar *ret = g_malloc0 (EVP_MAX_MD_SIZE);
-  g_assert (EVP_DigestFinal_ex (stream->priv->checksum, ret, &len));
-#elif defined(HAVE_GNUTLS)
-  guint len = gnutls_hash_get_len (stream->priv->checksum_type);
-  guchar *ret = g_malloc0 (len);
-  gnutls_hash_output (stream->priv->checksum, ret);
-#else
-  gsize len = g_checksum_type_get_length (stream->priv->checksum_type);
-  guchar *ret = g_malloc (len);
-  g_checksum_get_digest (stream->priv->checksum, ret, &len);
-#endif
-  if (ret_len)
-    *ret_len = len;
-  return ret;
-}
-
 char *
 ot_checksum_instream_get_string (OtChecksumInstream *stream)
 {
-#if defined(HAVE_OPENSSL)
-  unsigned len;
-  guint8 csum[EVP_MAX_MD_SIZE];
-  g_assert (EVP_DigestFinal_ex (stream->priv->checksum, csum, &len));
-  char *buf = g_malloc (len * 2 + 1);
-  ot_bin2hex (buf, (guint8*)csum, len);
-  return buf;
-#elif defined(HAVE_GNUTLS)
-  gsize len;
-  guint8 *csum = ot_checksum_instream_dup_digest(stream, &len);
-  char *buf = g_malloc0 (len * 2 + 1);
-  ot_bin2hex (buf, csum, len);
-  g_free (csum);
-  return buf;
-#else
-  return g_strdup (g_checksum_get_string (stream->priv->checksum));
-#endif
+  char buf[_OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&stream->priv->checksum, buf, sizeof(buf));
+  return g_strndup (buf, sizeof(buf));
 }

src/libotutil/ot-checksum-instream.h

@@ -52,12 +52,6 @@ GType          ot_checksum_instream_get_type     (void) G_GNUC_CONST;
 
 OtChecksumInstream * ot_checksum_instream_new          (GInputStream   *stream, GChecksumType   checksum);
 
-void   ot_checksum_instream_get_digest (OtChecksumInstream *stream,
-                                        guint8          *buffer,
-                                        gsize           *digest_len);
-
-guint8* ot_checksum_instream_dup_digest (OtChecksumInstream *stream,
-                                         gsize              *ret_len);
 char * ot_checksum_instream_get_string (OtChecksumInstream *stream);
 
 G_END_DECLS

src/libotutil/ot-checksum-utils.c

@@ -22,10 +22,15 @@
 #include "config.h"
 
 #include "otutil.h"
+#if defined(HAVE_OPENSSL)
+#include <openssl/evp.h>
+#elif defined(HAVE_GNUTLS)
+#include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
+#endif
 
 #include <string.h>
 
-
 void
 ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len)
 {
@@ -41,6 +46,120 @@ ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len)
   out_buf[j] = '\0';
 }
 
+/* I like to think of this as AbstractChecksumProxyFactoryBean. In homage to
+ * https://news.ycombinator.com/item?id=4549544
+ * aka http://static.springsource.org/spring/docs/2.5.x/api/org/springframework/aop/framework/AbstractSingletonProxyFactoryBean.html
+ */
+typedef struct {
+  gboolean initialized;
+#if defined(HAVE_OPENSSL)
+  EVP_MD_CTX *checksum;
+#elif defined(HAVE_GNUTLS)
+  gnutls_hash_hd_t checksum;
+#else
+  GChecksum *checksum;
+#endif
+  guint digest_len;
+} OtRealChecksum;
+
+G_STATIC_ASSERT (sizeof (OtChecksum) >= sizeof (OtRealChecksum));
+
+void
+ot_checksum_init (OtChecksum *checksum)
+{
+  OtRealChecksum *real = (OtRealChecksum*)checksum;
+  g_return_if_fail (!real->initialized);
+#if defined(HAVE_OPENSSL)
+  real->checksum = EVP_MD_CTX_create ();
+  g_assert (real->checksum);
+  g_assert (EVP_DigestInit_ex (real->checksum, EVP_sha256 (), NULL));
+  real->digest_len = EVP_MD_CTX_size (real->checksum);
+#elif defined(HAVE_GNUTLS)
+  g_assert (!gnutls_hash_init (&real->checksum, GNUTLS_DIG_SHA256));
+  real->digest_len = gnutls_hash_get_len (GNUTLS_DIG_SHA256);
+#else
+  real->checksum = g_checksum_new (G_CHECKSUM_SHA256);
+  real->digest_len = g_checksum_type_get_length (G_CHECKSUM_SHA256);
+#endif
+  g_assert_cmpint (real->digest_len, ==, _OSTREE_SHA256_DIGEST_LEN);
+  real->initialized = TRUE;
+}
+
+void
+ot_checksum_update (OtChecksum *checksum,
+                    const guint8   *buf,
+                    size_t          len)
+{
+  OtRealChecksum *real = (OtRealChecksum*)checksum;
+  g_return_if_fail (real->initialized);
+#if defined(HAVE_OPENSSL)
+  g_assert (EVP_DigestUpdate (real->checksum, buf, len));
+#elif defined(HAVE_GNUTLS)
+  g_assert (!gnutls_hash (real->checksum, buf, len));
+#else
+  g_checksum_update (real->checksum, buf, len);
+#endif
+}
+
+static void
+ot_checksum_get_digest_internal (OtRealChecksum *real,
+                                 guint8      *buf,
+                                 size_t       buflen)
+{
+  g_return_if_fail (real->initialized);
+  g_assert_cmpint (buflen, ==, _OSTREE_SHA256_DIGEST_LEN);
+#if defined(HAVE_OPENSSL)
+  guint digest_len = buflen;
+  g_assert (EVP_DigestFinal_ex (real->checksum, buf, &digest_len));
+  g_assert_cmpint (digest_len, ==, buflen);
+#elif defined(HAVE_GNUTLS)
+  gnutls_hash_output (real->checksum, buf);
+#else
+  gsize digest_len = buflen;
+  g_checksum_get_digest (real->checksum, buf, &digest_len);
+  g_assert_cmpint (digest_len, ==, buflen);
+#endif
+}
+
+void
+ot_checksum_get_digest (OtChecksum *checksum,
+                        guint8      *buf,
+                        size_t       buflen)
+{
+  OtRealChecksum *real = (OtRealChecksum*)checksum;
+  ot_checksum_get_digest_internal (real, buf, buflen);
+  real->initialized = FALSE;
+}
+
+void
+ot_checksum_get_hexdigest (OtChecksum *checksum,
+                           char           *buf,
+                           size_t          buflen)
+{
+  OtRealChecksum *real = (OtRealChecksum*)checksum;
+  const guint digest_len = real->digest_len;
+  guint8 digest_buf[digest_len];
+  ot_checksum_get_digest (checksum, digest_buf, digest_len);
+  ot_bin2hex (buf, (guint8*)digest_buf, digest_len);
+  real->initialized = FALSE;
+}
+
+void
+ot_checksum_clear (OtChecksum *checksum)
+{
+  OtRealChecksum *real = (OtRealChecksum*)checksum;
+  if (!real->initialized)
+    return;
+#if defined(HAVE_OPENSSL)
+  EVP_MD_CTX_destroy (real->checksum);
+#elif defined(HAVE_GNUTLS)
+  gnutls_hash_deinit (real->checksum, NULL);
+#else
+  g_checksum_free (real->checksum);
+#endif
+  real->initialized = FALSE;
+}
+
 guchar *
 ot_csum_from_gchecksum (GChecksum  *checksum)
 {
@@ -57,7 +176,7 @@ ot_gio_write_update_checksum (GOutputStream  *out,
                               gconstpointer   data,
                               gsize           len,
                               gsize          *out_bytes_written,
-                              GChecksum      *checksum,
+                              OtChecksum     *checksum,
                               GCancellable   *cancellable,
                               GError        **error)
 {
@@ -73,14 +192,14 @@ ot_gio_write_update_checksum (GOutputStream  *out,
     }
 
   if (checksum)
-    g_checksum_update (checksum, data, len);
+    ot_checksum_update (checksum, data, len);
   return TRUE;
 }
 
 gboolean
 ot_gio_splice_update_checksum (GOutputStream  *out,
                                GInputStream   *in,
-                               GChecksum      *checksum,
+                               OtChecksum     *checksum,
                                GCancellable   *cancellable,
                                GError        **error)
 {
@@ -92,7 +211,7 @@ ot_gio_splice_update_checksum (GOutputStream  *out,
       char buf[4096];
       do
         {
-          if (!g_input_stream_read_all (in, buf, sizeof(buf), &bytes_read, cancellable, error))
+          if (!g_input_stream_read_all (in, buf, sizeof (buf), &bytes_read, cancellable, error))
             return FALSE;
           if (!ot_gio_write_update_checksum (out, buf, bytes_read, &bytes_written, checksum,
                                              cancellable, error))
@@ -119,27 +238,20 @@ ot_gio_splice_get_checksum (GOutputStream  *out,
                             GCancellable   *cancellable,
                             GError        **error)
 {
-  g_autoptr(GChecksum) checksum = g_checksum_new (G_CHECKSUM_SHA256);
+  g_auto(OtChecksum) checksum = { 0, };
+  ot_checksum_init (&checksum);
 
-  if (!ot_gio_splice_update_checksum (out, in, checksum, cancellable, error))
+  if (!ot_gio_splice_update_checksum (out, in, &checksum, cancellable, error))
     return FALSE;
 
-  g_autofree guchar *ret_csum = ot_csum_from_gchecksum (checksum);
+  guint8 digest[_OSTREE_SHA256_DIGEST_LEN];
+  ot_checksum_get_digest (&checksum, digest, sizeof (digest));
+  g_autofree guchar *ret_csum = g_malloc (sizeof (digest));
+  memcpy (ret_csum, digest, sizeof (digest));
   ot_transfer_out_value (out_csum, &ret_csum);
   return TRUE;
 }
 
-gboolean
-ot_gio_checksum_stream (GInputStream   *in,
-                        guchar        **out_csum,
-                        GCancellable   *cancellable,
-                        GError        **error)
-{
-  if (!out_csum)
-    return TRUE;
-  return ot_gio_splice_get_checksum (NULL, in, out_csum, cancellable, error);
-}
-
 char *
 ot_checksum_file_at (int             dfd,
                      const char     *path,
@@ -151,9 +263,12 @@ ot_checksum_file_at (int             dfd,
   if (!ot_openat_read_stream (dfd, path, TRUE, &in, cancellable, error))
     return FALSE;
 
-  g_autoptr(GChecksum) checksum = g_checksum_new (checksum_type);
-  if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+  g_auto(OtChecksum) checksum = { 0, };
+  ot_checksum_init (&checksum);
+  if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error))
     return FALSE;
 
-  return g_strdup (g_checksum_get_string (checksum));
+  char hexdigest[_OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&checksum, hexdigest, sizeof (hexdigest));
+  return g_strdup (hexdigest);
 }

src/libotutil/ot-checksum-utils.h

@@ -21,7 +21,7 @@
 
 #pragma once
 
-#include <gio/gio.h>
+#include "libglnx.h"
 
 G_BEGIN_DECLS
 
@@ -29,11 +29,50 @@ void ot_bin2hex (char *out_buf, const guint8 *inbuf, gsize len);
 
 guchar *ot_csum_from_gchecksum (GChecksum *checksum);
 
+struct OtChecksum {
+  gboolean initialized;
+  guint uints[2];
+  gpointer data[2];
+};
+typedef struct OtChecksum OtChecksum;
+
+/* Same as OSTREE_SHA256_DIGEST_LEN, but this header can't depend on that */
+#define _OSTREE_SHA256_DIGEST_LEN (32)
+#if defined(OSTREE_SHA256_DIGEST_LEN) && _OSTREE_SHA256_DIGEST_LEN != OSTREE_SHA256_DIGEST_LEN
+#error Mismatched OSTREE_SHA256_DIGEST_LEN
+#endif
+/* See above */
+#define _OSTREE_SHA256_STRING_LEN (64)
+#if defined(OSTREE_SHA256_STRING_LEN) && _OSTREE_SHA256_STRING_LEN != OSTREE_SHA256_STRING_LEN
+#error Mismatched OSTREE_SHA256_STRING_LEN
+#endif
+
+void ot_checksum_init (OtChecksum *checksum);
+void ot_checksum_update (OtChecksum *checksum,
+                         const guint8   *buf,
+                         size_t          len);
+static inline void
+ot_checksum_update_bytes (OtChecksum *checksum,
+                          GBytes     *buf)
+{
+  gsize len;
+  const guint8 *bufdata = g_bytes_get_data (buf, &len);
+  ot_checksum_update (checksum, bufdata, len);
+}
+void ot_checksum_get_digest (OtChecksum *checksum,
+                             guint8      *buf,
+                             size_t       buflen);
+void ot_checksum_get_hexdigest (OtChecksum *checksum,
+                                char           *buf,
+                                size_t          buflen);
+void ot_checksum_clear (OtChecksum *checksum);
+G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtChecksum, ot_checksum_clear)
+
 gboolean ot_gio_write_update_checksum (GOutputStream  *out,
                                        gconstpointer   data,
                                        gsize           len,
                                        gsize          *out_bytes_written,
-                                       GChecksum      *checksum,
+                                       OtChecksum     *checksum,
                                        GCancellable   *cancellable,
                                        GError        **error);
 
@@ -45,15 +84,10 @@ gboolean ot_gio_splice_get_checksum (GOutputStream  *out,
 
 gboolean ot_gio_splice_update_checksum (GOutputStream  *out,
                                         GInputStream   *in,
-                                        GChecksum      *checksum,
+                                        OtChecksum     *checksum,
                                         GCancellable   *cancellable,
                                         GError        **error);
 
-gboolean ot_gio_checksum_stream (GInputStream   *in,
-                                 guchar        **out_csum,
-                                 GCancellable   *cancellable,
-                                 GError        **error);
-
 char * ot_checksum_file_at (int             dfd,
                             const char     *path,
                             GChecksumType   checksum_type,

src/libotutil/ot-fs-utils.c

@@ -22,6 +22,7 @@
 #include "ot-fs-utils.h"
 #include "libglnx.h"
 #include <sys/xattr.h>
+#include <sys/mman.h>
 #include <gio/gunixinputstream.h>
 #include <gio/gunixoutputstream.h>
 
@@ -35,6 +36,9 @@ ot_fdrel_to_gfile (int dfd, const char *path)
   return g_file_new_for_path (abspath);
 }
 
+/* Wraps readlinkat(), and sets the `symlink-target` property
+ * of @target_info.
+ */
 gboolean
 ot_readlinkat_gfile_info (int             dfd,
                           const char     *path,
@@ -53,7 +57,6 @@ ot_readlinkat_gfile_info (int             dfd,
   return TRUE;
 }
 
-
 /**
  * ot_openat_read_stream:
  * @dfd: Directory file descriptor
@@ -76,16 +79,10 @@ ot_openat_read_stream (int             dfd,
                        GCancellable   *cancellable,
                        GError        **error)
 {
-  int fd = -1;
-  int flags = O_RDONLY | O_NOCTTY | O_CLOEXEC;
-
-  if (!follow)
-    flags |= O_NOFOLLOW;
-
-  if (TEMP_FAILURE_RETRY (fd = openat (dfd, path, flags, 0)) < 0)
-    return glnx_throw_errno_prefix (error, "openat(%s)", path);
-
-  *out_istream = g_unix_input_stream_new (fd, TRUE);
+  glnx_autofd int fd = -1;
+  if (!glnx_openat_rdonly (dfd, path, follow, &fd, error))
+    return FALSE;
+  *out_istream = g_unix_input_stream_new (glnx_steal_fd (&fd), TRUE);
   return TRUE;
 }
 
@@ -130,7 +127,7 @@ ot_dfd_iter_init_allow_noent (int dfd,
                               gboolean *out_exists,
                               GError **error)
 {
-  glnx_fd_close int fd = glnx_opendirat_with_errno (dfd, path, TRUE);
+  glnx_autofd int fd = glnx_opendirat_with_errno (dfd, path, TRUE);
   if (fd < 0)
     {
       if (errno != ENOENT)
@@ -144,22 +141,57 @@ ot_dfd_iter_init_allow_noent (int dfd,
   return TRUE;
 }
 
-GBytes *
-ot_file_mapat_bytes (int dfd,
-                     const char *path,
-                     GError **error)
+typedef struct {
+  gpointer addr;
+  gsize len;
+} MapData;
+
+static void
+map_data_destroy (gpointer data)
 {
-  glnx_fd_close int fd = openat (dfd, path, O_RDONLY | O_CLOEXEC);
-  g_autoptr(GMappedFile) mfile = NULL;
+  MapData *mdata = data;
+  (void) munmap (mdata->addr, mdata->len);
+  g_free (mdata);
+}
 
-  if (fd < 0)
-    return glnx_null_throw_errno_prefix (error, "openat(%s)", path);
-
-  mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
-  if (!mfile)
+/* Return a newly-allocated GBytes that refers to the contents of the file
+ * starting at offset @start. If the file is large enough, mmap() may be used.
+ */
+GBytes *
+ot_fd_readall_or_mmap (int           fd,
+                       goffset       start,
+                       GError      **error)
+{
+  struct stat stbuf;
+  if (!glnx_fstat (fd, &stbuf, error))
     return FALSE;
 
-  return g_mapped_file_get_bytes (mfile);
+  /* http://stackoverflow.com/questions/258091/when-should-i-use-mmap-for-file-access */
+  if (start > stbuf.st_size)
+    return g_bytes_new_static (NULL, 0);
+  const gsize len = stbuf.st_size - start;
+  if (len > 16*1024)
+    {
+      /* The reason we don't use g_mapped_file_new_from_fd() here
+       * is it doesn't support passing an offset, which is actually
+       * used by the static delta code.
+       */
+      gpointer map = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, start);
+      if (map == (void*)-1)
+        return glnx_null_throw_errno_prefix (error, "mmap");
+
+      MapData *mdata = g_new (MapData, 1);
+      mdata->addr = map;
+      mdata->len = len;
+
+      return g_bytes_new_with_free_func (map, len, map_data_destroy, mdata);
+    }
+
+  /* Fall through to plain read into a malloc buffer */
+  if (lseek (fd, start, SEEK_SET) < 0)
+    return glnx_null_throw_errno_prefix (error, "lseek");
+  /* Not cancellable since this should be small */
+  return glnx_fd_readall_bytes (fd, NULL, error);
 }
 
 /* Given an input stream, splice it to an anonymous file (O_TMPFILE).

src/libotutil/ot-fs-utils.h

@@ -85,8 +85,7 @@ ot_map_anonymous_tmpfile_from_content (GInputStream *instream,
                                        GCancellable *cancellable,
                                        GError      **error);
 
-GBytes *ot_file_mapat_bytes (int dfd,
-                             const char *path,
-                             GError **error);
+GBytes *ot_fd_readall_or_mmap (int fd, goffset offset,
+                               GError **error);
 
 G_END_DECLS

src/libotutil/ot-unix-utils.c

@@ -34,22 +34,21 @@
 #include <stdlib.h>
 #include <errno.h>
 
+/* Ensure that a pathname component @name does not contain the special Unix
+ * entries `.` or `..`, and does not contain `/`.
+ */
 gboolean
 ot_util_filename_validate (const char *name,
                            GError    **error)
 {
   if (strcmp (name, ".") == 0)
-    {
-      return glnx_throw (error, "Invalid self-referential filename '.'");
-    }
+    return glnx_throw (error, "Invalid self-referential filename '.'");
   if (strcmp (name, "..") == 0)
-    {
-      return glnx_throw (error, "Invalid path uplink filename '..'");
-    }
+    return glnx_throw (error, "Invalid path uplink filename '..'");
   if (strchr (name, '/') != NULL)
-    {
-      return glnx_throw (error, "Invalid / in filename %s", name);
-    }
+    return glnx_throw (error, "Invalid / in filename %s", name);
+  if (!g_utf8_validate (name, -1, NULL))
+    return glnx_throw (error, "Invalid UTF-8 in filename %s", name);
   return TRUE;
 }
 
@@ -58,8 +57,8 @@ ot_split_string_ptrarray (const char *str,
                           char        c)
 {
   GPtrArray *ret = g_ptr_array_new_with_free_func (g_free);
-  const char *p;
 
+  const char *p;
   do {
     p = strchr (str, '/');
     if (!p)
@@ -77,40 +76,29 @@ ot_split_string_ptrarray (const char *str,
   return ret;
 }
 
+/* Given a pathname @path, split it into individual entries in @out_components,
+ * validating that it does not have backreferences (`..`) etc.
+ */
 gboolean
 ot_util_path_split_validate (const char *path,
                              GPtrArray **out_components,
                              GError    **error)
 {
-  gboolean ret = FALSE;
-  int i;
-  g_autoptr(GPtrArray) ret_components = NULL;
-
   if (strlen (path) > PATH_MAX)
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Path '%s' is too long", path);
-      goto out;
-    }
+    return glnx_throw (error, "Path '%s' is too long", path);
 
-  ret_components = ot_split_string_ptrarray (path, '/');
+  g_autoptr(GPtrArray) ret_components = ot_split_string_ptrarray (path, '/');
 
   /* Canonicalize by removing '.' and '', throw an error on .. */
-  for (i = ret_components->len-1; i >= 0; i--)
+  for (int i = ret_components->len-1; i >= 0; i--)
     {
       const char *name = ret_components->pdata[i];
       if (strcmp (name, "..") == 0)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Invalid uplink '..' in path %s", path);
-          goto out;
-        }
+        return glnx_throw (error, "Invalid uplink '..' in path %s", path);
       if (strcmp (name, ".") == 0 || name[0] == '\0')
         g_ptr_array_remove_index (ret_components, i);
     }
 
-  ret = TRUE;
   ot_transfer_out_value(out_components, &ret_components);
- out:
-  return ret;
+  return TRUE;
 }

src/libotutil/ot-variant-builder.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2017 Alexander Larsson <alexl@redhat.com>.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Author: Alexander Larsson <alexl@redhat.com>.
+ */
+
+#pragma once
+
+#include <gio/gio.h>
+
+#include "libglnx.h"
+
+G_BEGIN_DECLS
+
+typedef struct _OtVariantBuilder OtVariantBuilder;
+
+OtVariantBuilder *ot_variant_builder_new         (const GVariantType  *type,
+                                                  int                  fd);
+void              ot_variant_builder_unref       (OtVariantBuilder    *builder);
+OtVariantBuilder *ot_variant_builder_ref         (OtVariantBuilder    *builder);
+gboolean          ot_variant_builder_end         (OtVariantBuilder    *builder,
+                                                  GError             **error);
+gboolean          ot_variant_builder_open        (OtVariantBuilder    *builder,
+                                                  const GVariantType  *type,
+                                                  GError             **error);
+gboolean          ot_variant_builder_close       (OtVariantBuilder    *builder,
+                                                  GError             **error);
+gboolean          ot_variant_builder_add_from_fd (OtVariantBuilder    *builder,
+                                                  const GVariantType  *type,
+                                                  int                  fd,
+                                                  guint64              size,
+                                                  GError             **error);
+gboolean          ot_variant_builder_add_value   (OtVariantBuilder    *builder,
+                                                  GVariant            *value,
+                                                  GError             **error);
+gboolean          ot_variant_builder_add         (OtVariantBuilder    *builder,
+                                                  GError             **error,
+                                                  const gchar         *format_string,
+                                                  ...);
+void              ot_variant_builder_add_parsed  (OtVariantBuilder    *builder,
+                                                  const gchar         *format,
+                                                  ...);
+
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OtVariantBuilder, ot_variant_builder_unref)
+
+G_END_DECLS

src/libotutil/ot-variant-utils.c

@@ -25,10 +25,10 @@
 #include <gio/gfiledescriptorbased.h>
 
 #include <string.h>
-#include <sys/mman.h>
 
 #include "otutil.h"
 
+/* Create a new GVariant empty GVariant of type a{sv} */
 GVariant *
 ot_gvariant_new_empty_string_dict (void)
 {
@@ -37,170 +37,69 @@ ot_gvariant_new_empty_string_dict (void)
   return g_variant_builder_end (&builder);
 }
 
+
+/* Create a new GVariant of type ay from the raw @data pointer */
 GVariant *
 ot_gvariant_new_bytearray (const guchar   *data,
                            gsize           len)
 {
-  gpointer data_copy;
-  GVariant *ret;
-
-  data_copy = g_memdup (data, len);
-  ret = g_variant_new_from_data (G_VARIANT_TYPE ("ay"), data_copy,
+  gpointer data_copy = g_memdup (data, len);
+  GVariant *ret = g_variant_new_from_data (G_VARIANT_TYPE ("ay"), data_copy,
                                  len, FALSE, g_free, data_copy);
   return ret;
 }
 
+/* Convert a GBytes into a GVariant of type ay (byte array) */
 GVariant *
 ot_gvariant_new_ay_bytes (GBytes *bytes)
 {
   gsize size;
-  gconstpointer data;
-  data = g_bytes_get_data (bytes, &size);
+  gconstpointer data = g_bytes_get_data (bytes, &size);
   g_bytes_ref (bytes);
   return g_variant_new_from_data (G_VARIANT_TYPE ("ay"), data, size,
                                   TRUE, (GDestroyNotify)g_bytes_unref, bytes);
 }
 
-GHashTable *
-ot_util_variant_asv_to_hash_table (GVariant *variant)
-{
-  GHashTable *ret;
-  GVariantIter *viter;
-  char *key;
-  GVariant *value;
-  
-  ret = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)g_variant_unref);
-  viter = g_variant_iter_new (variant);
-  while (g_variant_iter_next (viter, "{s@v}", &key, &value))
-    g_hash_table_replace (ret, key, g_variant_ref_sink (value));
-  
-  g_variant_iter_free (viter);
-  
-  return ret;
-}
-
-GVariant *
-ot_util_variant_take_ref (GVariant *variant)
-{
-  return g_variant_take_ref (variant);
-}
-
+/* Create a GVariant in @out_variant that is backed by
+ * the data from @fd, starting at @start.  If the data is
+ * large enough, mmap() may be used.  @trusted is used
+ * by the GVariant core; see g_variant_new_from_data().
+ */
 gboolean
-ot_util_variant_map_at (int dfd,
-                        const char *path,
-                        const GVariantType *type,
-                        OtVariantMapFlags flags,
-                        GVariant **out_variant,
-                        GError  **error)
+ot_variant_read_fd (int                    fd,
+                    goffset                start,
+                    const GVariantType    *type,
+                    gboolean               trusted,
+                    GVariant             **out_variant,
+                    GError               **error)
 {
-  glnx_fd_close int fd = -1;
-  const gboolean trusted = (flags & OT_VARIANT_MAP_TRUSTED) > 0;
+  g_autoptr(GBytes) bytes = ot_fd_readall_or_mmap (fd, start, error);
+  if (!bytes)
+    return FALSE;
 
-  fd = openat (dfd, path, O_RDONLY | O_CLOEXEC);
-  if (fd < 0)
-    {
-      if (errno == ENOENT && (flags & OT_VARIANT_MAP_ALLOW_NOENT) > 0)
-        {
-          *out_variant = NULL;
-          return TRUE;
-        }
-      else
-        {
-          glnx_set_error_from_errno (error);
-          g_prefix_error (error, "Opening %s: ", path);
-          return FALSE;
-        }
-    }
-
-  return ot_util_variant_map_fd (fd, 0, type, trusted, out_variant, error);
-}
-
-typedef struct {
-  gpointer addr;
-  gsize len;
-} VariantMapData;
-
-static void
-variant_map_data_destroy (gpointer data)
-{
-  VariantMapData *mdata = data;
-  (void) munmap (mdata->addr, mdata->len);
-  g_free (mdata);
-}
-
-gboolean
-ot_util_variant_map_fd (int                    fd,
-                        goffset                start,
-                        const GVariantType    *type,
-                        gboolean               trusted,
-                        GVariant             **out_variant,
-                        GError               **error)
-{
-  gboolean ret = FALSE;
-  gpointer map;
-  struct stat stbuf;
-  VariantMapData *mdata = NULL;
-  gsize len;
-
-  if (fstat (fd, &stbuf) != 0)
-    {
-      glnx_set_error_from_errno (error);
-      goto out;
-    }
-
-  len = stbuf.st_size - start;
-  map = mmap (NULL, len, PROT_READ, MAP_PRIVATE, fd, start);
-  if (!map)
-    {
-      glnx_set_error_from_errno (error);
-      goto out;
-    }
-
-  mdata = g_new (VariantMapData, 1);
-  mdata->addr = map;
-  mdata->len = len;
-
-  ret = TRUE;
-  *out_variant = g_variant_ref_sink (g_variant_new_from_data (type, map, len, trusted,
-                                                              variant_map_data_destroy, mdata));
- out:
-  return ret;
-}
-
-GInputStream *
-ot_variant_read (GVariant             *variant)
-{
-  GMemoryInputStream *ret = NULL;
-
-  ret = (GMemoryInputStream*)g_memory_input_stream_new_from_data (g_variant_get_data (variant),
-                                                                  g_variant_get_size (variant),
-                                                                  NULL);
-  g_object_set_data_full ((GObject*)ret, "ot-variant-data",
-                          g_variant_ref (variant), (GDestroyNotify) g_variant_unref);
-  return (GInputStream*)ret;
+  *out_variant = g_variant_ref_sink (g_variant_new_from_bytes (type, bytes, trusted));
+  return TRUE;
 }
 
+/* GVariants are immutable; this function allows generating an open builder
+ * for a new variant, inherting the data from @variant.
+ */
 GVariantBuilder *
 ot_util_variant_builder_from_variant (GVariant            *variant,
                                       const GVariantType  *type)
 {
-  GVariantBuilder *builder = NULL;
-  
-  builder = g_variant_builder_new (type);
-  
+  GVariantBuilder *builder = g_variant_builder_new (type);
+
   if (variant != NULL)
     {
-      gint i, n;
-
-      n = g_variant_n_children (variant);
-      for (i = 0; i < n; i++)
+      const int n = g_variant_n_children (variant);
+      for (int i = 0; i < n; i++)
         {
-          GVariant *child = g_variant_get_child_value (variant, i);
+          g_autoptr(GVariant) child = g_variant_get_child_value (variant, i);
           g_variant_builder_add_value (builder, child);
-          g_variant_unref (child);
         }
     }
-    
+
   return builder;
 }
 
@@ -221,28 +120,23 @@ ot_variant_bsearch_str (GVariant   *array,
                         const char *str,
                         int        *out_pos)
 {
-  gsize imax, imin;
-  gsize imid = -1;
-  gsize n;
-
-  n = g_variant_n_children (array);
+  const gsize n = g_variant_n_children (array);
   if (n == 0)
     return FALSE;
 
-  imax = n - 1;
-  imin = 0;
+  gsize imax = n - 1;
+  gsize imin = 0;
+  gsize imid = -1;
   while (imax >= imin)
     {
-      g_autoptr(GVariant) child = NULL;
       const char *cur;
-      int cmp;
 
       imid = (imin + imax) / 2;
 
-      child = g_variant_get_child_value (array, imid);
+      g_autoptr(GVariant) child = g_variant_get_child_value (array, imid);
       g_variant_get_child (child, 0, "&s", &cur, NULL);
 
-      cmp = strcmp (cur, str);
+      int cmp = strcmp (cur, str);
       if (cmp < 0)
         imin = imid + 1;
       else if (cmp > 0)

src/libotutil/ot-variant-utils.h

@@ -32,30 +32,12 @@ GVariant *ot_gvariant_new_ay_bytes (GBytes *bytes);
 
 GVariant *ot_gvariant_new_empty_string_dict (void);
 
-GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
-
-GVariant * ot_util_variant_take_ref (GVariant *variant);
-
-typedef enum {
-  OT_VARIANT_MAP_TRUSTED = (1 << 0),
-  OT_VARIANT_MAP_ALLOW_NOENT = (1 << 1)
-} OtVariantMapFlags;
-
-gboolean ot_util_variant_map_at (int dfd,
-                                 const char *path,
-                                 const GVariantType *type,
-                                 OtVariantMapFlags flags,
-                                 GVariant **out_variant,
-                                 GError  **error);
-
-gboolean ot_util_variant_map_fd (int                  fd,
-                                 goffset              offset,
-                                 const GVariantType  *type,
-                                 gboolean             trusted,
-                                 GVariant           **out_variant,
-                                 GError             **error);
-
-GInputStream *ot_variant_read (GVariant             *variant);
+gboolean ot_variant_read_fd (int                  fd,
+                             goffset              offset,
+                             const GVariantType  *type,
+                             gboolean             trusted,
+                             GVariant           **out_variant,
+                             GError             **error);
 
 GVariantBuilder *ot_util_variant_builder_from_variant (GVariant            *variant,
                                                        const GVariantType  *type);

src/libotutil/otutil.c

@@ -1,40 +0,0 @@
-/*
- * Copyright (C) 2011 Colin Walters <walters@verbum.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- * Boston, MA 02111-1307, USA.
- *
- * Author: Colin Walters <walters@verbum.org>
- */
-
-#include "config.h"
-
-#include "otutil.h"
-
-#include <string.h>
-
-void
-ot_ptrarray_add_many (GPtrArray  *a, ...)
-{
-  va_list args;
-  void *p;
-
-  va_start (args, a);
-
-  while ((p = va_arg (args, void *)) != NULL)
-    g_ptr_array_add (a, p);
-
-  va_end (args);
-}

src/libotutil/otutil.h

@@ -48,8 +48,7 @@
 #include <ot-opt-utils.h>
 #include <ot-unix-utils.h>
 #include <ot-variant-utils.h>
+#include <ot-variant-builder.h>
 #include <ot-checksum-utils.h>
 #include <ot-gpg-utils.h>
 #include <ot-checksum-instream.h>
-
-void ot_ptrarray_add_many (GPtrArray  *a, ...) G_GNUC_NULL_TERMINATED; 

src/ostree/main.c

@@ -32,37 +32,94 @@
 #include "ot-builtins.h"
 
 static OstreeCommand commands[] = {
-  { "admin", ostree_builtin_admin },
-  { "cat", ostree_builtin_cat },
-  { "checkout", ostree_builtin_checkout },
-  { "checksum", ostree_builtin_checksum },
-  { "commit", ostree_builtin_commit },
-  { "config", ostree_builtin_config },
-  { "diff", ostree_builtin_diff },
-  { "export", ostree_builtin_export },
+  /* Note: all admin related commands have
+   * no_repo as their command flag, but each
+   * admin command may have their own
+   * admin flag
+   */
+  { "admin", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ostree_builtin_admin,
+    "Commands for managing a host system booted with ostree" },
+  { "cat", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_cat,
+    "Concatenate contents of files"},
+  { "checkout", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_checkout,
+    "Check out a commit into a filesystem tree" },
+  { "checksum", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ostree_builtin_checksum,
+    "Checksum a file or directory" },
+  { "commit", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_commit,
+    "Commit a new revision" },
+  { "config", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_config,
+    "Change repo configuration settings" },
+  { "diff", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_diff,
+    "Compare directory TARGETDIR against revision REV"},
+  { "export", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_export,
+    "Stream COMMIT to stdout in tar format" },
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
-  { "find-remotes", ostree_builtin_find_remotes },
-  { "create-usb", ostree_builtin_create_usb },
+  { "find-remotes", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_find_remotes,
+    "Find remotes to serve the given refs" },
+  { "create-usb", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_create_usb,
+    "Copy the refs to a USB stick" },
 #endif
-  { "fsck", ostree_builtin_fsck },
-  { "gpg-sign", ostree_builtin_gpg_sign },
-  { "init", ostree_builtin_init },
-  { "log", ostree_builtin_log },
-  { "ls", ostree_builtin_ls },
-  { "prune", ostree_builtin_prune },
-  { "pull-local", ostree_builtin_pull_local },
-#ifdef HAVE_LIBSOUP 
-  { "pull", ostree_builtin_pull },
+  { "fsck", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_fsck,
+    "Check the repository for consistency" },
+  { "gpg-sign", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_gpg_sign,
+    "Sign a commit" },
+  { "init", OSTREE_BUILTIN_FLAG_NO_CHECK,
+    ostree_builtin_init,
+    "Initialize a new empty repository" },
+  { "log", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_log,
+    "Show log starting at commit or ref" },
+  { "ls", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_ls,
+    "List file paths" },
+  { "prune", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_prune,
+    "Search for unreachable objects" },
+  { "pull-local", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_pull_local,
+    "Copy data from SRC_REPO" },
+#ifdef HAVE_LIBCURL_OR_LIBSOUP
+  { "pull", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_pull,
+    "Download data from remote repository" },
 #endif
-  { "refs", ostree_builtin_refs },
-  { "remote", ostree_builtin_remote },
-  { "reset", ostree_builtin_reset },
-  { "rev-parse", ostree_builtin_rev_parse },
-  { "show", ostree_builtin_show },
-  { "static-delta", ostree_builtin_static_delta },
-  { "summary", ostree_builtin_summary },
+  { "refs", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_refs,
+    "List refs" },
+  { "remote", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ostree_builtin_remote,
+    "Remote commands that may involve internet access" },
+  { "reset", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_reset,
+    "Reset a REF to a previous COMMIT" },
+  { "rev-parse", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_rev_parse,
+    "Output the target of a rev" },
+  { "show", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_show,
+    "Output a metadata object" },
+  { "static-delta", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_static_delta,
+    "Static delta related commands" },
+  { "summary", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_summary,
+    "Manage summary metadata" },
 #if defined(HAVE_LIBSOUP) && defined(BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE)
-  { "trivial-httpd", ostree_builtin_trivial_httpd },
+  { "trivial-httpd", OSTREE_BUILTIN_FLAG_NONE,
+    ostree_builtin_trivial_httpd,
+    NULL },
 #endif
   { NULL }
 };

src/ostree/ostree-trivial-httpd.c

@@ -353,7 +353,7 @@ do_get (OtTrivialHttpd    *self,
       
       if (msg->method == SOUP_METHOD_GET)
         {
-          glnx_fd_close int fd = -1;
+          glnx_autofd int fd = -1;
           g_autoptr(GMappedFile) mapping = NULL;
           gsize buffer_length, file_size;
           SoupRange *ranges;

src/ostree/ot-admin-builtin-cleanup.c

@@ -38,17 +38,17 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_cleanup (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_cleanup (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
   gboolean ret = FALSE;
 
-  context = g_option_context_new ("Delete untagged deployments and repository objects");
+  context = g_option_context_new ("");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   if (!ostree_sysroot_cleanup (sysroot, cancellable, error))

src/ostree/ot-admin-builtin-deploy.c

@@ -60,17 +60,17 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_deploy (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = NULL;
+  g_autoptr(OstreeKernelArgs) kargs = NULL;
 
   g_autoptr(GOptionContext) context =
-    g_option_context_new ("REFSPEC - Checkout revision REFSPEC as the new default deployment");
+    g_option_context_new ("REFSPEC");
 
   g_autoptr(OstreeSysroot) sysroot = NULL;
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     return FALSE;
 
   if (argc < 2)

src/ostree/ot-admin-builtin-diff.c

@@ -41,7 +41,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_diff (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
@@ -54,13 +54,13 @@ ot_admin_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError
   g_autoptr(GFile) orig_etc_path = NULL;
   g_autoptr(GFile) new_etc_path = NULL;
 
-  context = g_option_context_new ("Diff current /etc configuration versus default");
+  context = g_option_context_new ("");
 
   g_option_context_add_main_entries (context, options, NULL);
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   if (!ot_admin_require_booted_deployment_or_osname (sysroot, opt_osname,

src/ostree/ot-admin-builtin-init-fs.c

@@ -38,15 +38,15 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_init_fs (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_init_fs (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = g_option_context_new ("PATH - Initialize a root filesystem");
+  g_autoptr(GOptionContext) context = g_option_context_new ("PATH");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER |
-                                           OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED |
-                                           OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT,
-                                          NULL, cancellable, error))
+                                          OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED |
+                                          OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT,
+                                          invocation, NULL, cancellable, error))
     return FALSE;
 
   if (argc < 2)
@@ -57,7 +57,7 @@ ot_admin_builtin_init_fs (int argc, char **argv, GCancellable *cancellable, GErr
 
   const char *sysroot_path = argv[1];
 
-  glnx_fd_close int root_dfd = -1;
+  glnx_autofd int root_dfd = -1;
   if (!glnx_opendirat (AT_FDCWD, sysroot_path, TRUE, &root_dfd, error))
     return FALSE;
 

src/ostree/ot-admin-builtin-instutil.c

@@ -28,31 +28,35 @@
 
 #include <glib/gi18n.h>
 
-typedef struct {
-  const char *name;
-  gboolean (*fn) (int argc, char **argv, GCancellable *cancellable, GError **error);
-} OstreeAdminInstUtilCommand;
-
-static OstreeAdminInstUtilCommand admin_instutil_subcommands[] = {
+static OstreeCommand admin_instutil_subcommands[] = {
 #ifdef HAVE_SELINUX
-  { "selinux-ensure-labeled", ot_admin_instutil_builtin_selinux_ensure_labeled },
+  { "selinux-ensure-labeled", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_instutil_builtin_selinux_ensure_labeled,
+    "Relabel all or part of a deployment" },
 #endif
-  { "set-kargs", ot_admin_instutil_builtin_set_kargs },
-  { "grub2-generate", ot_admin_instutil_builtin_grub2_generate },
-  { NULL, NULL }
+  { "set-kargs", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_instutil_builtin_set_kargs,
+    "Set new kernel command line arguments(Not stable)"  },
+  { "grub2-generate", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_instutil_builtin_grub2_generate,
+    "Generate GRUB2 configuration from given BLS entries" },
+  { NULL, 0, NULL, NULL }
 };
 
 static GOptionContext *
 ostree_admin_instutil_option_context_new_with_commands (void)
 {
-  OstreeAdminInstUtilCommand *command = admin_instutil_subcommands;
+  OstreeCommand *command = admin_instutil_subcommands;
   GOptionContext *context = g_option_context_new ("COMMAND");
 
   g_autoptr(GString) summary = g_string_new ("Builtin \"admin instutil\" Commands:");
 
   while (command->name != NULL)
     {
-      g_string_append_printf (summary, "\n  %s", command->name);
+      g_string_append_printf (summary, "\n  %-24s", command->name);
+        if (command->description != NULL)
+          g_string_append_printf (summary, "%s", command->description);
+
       command++;
     }
 
@@ -62,7 +66,7 @@ ostree_admin_instutil_option_context_new_with_commands (void)
 }
 
 gboolean
-ot_admin_builtin_instutil (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_instutil (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   const char *subcommand_name = NULL;
   int in, out;
@@ -90,7 +94,7 @@ ot_admin_builtin_instutil (int argc, char **argv, GCancellable *cancellable, GEr
 
   argc = out;
 
-  OstreeAdminInstUtilCommand *subcommand = admin_instutil_subcommands;
+  OstreeCommand *subcommand = admin_instutil_subcommands;
   while (subcommand->name)
     {
       if (g_strcmp0 (subcommand_name, subcommand->name) == 0)
@@ -106,7 +110,7 @@ ot_admin_builtin_instutil (int argc, char **argv, GCancellable *cancellable, GEr
       /* This will not return for some options (e.g. --version). */
       if (ostree_admin_option_context_parse (context, NULL, &argc, &argv,
                                              OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT,
-                                             NULL, cancellable, error))
+                                             invocation, NULL, cancellable, error))
         {
           if (subcommand_name == NULL)
             {
@@ -128,7 +132,8 @@ ot_admin_builtin_instutil (int argc, char **argv, GCancellable *cancellable, GEr
   g_autofree char *prgname = g_strdup_printf ("%s %s", g_get_prgname (), subcommand_name);
   g_set_prgname (prgname);
 
-  if (!subcommand->fn (argc, argv, cancellable, error))
+  OstreeCommandInvocation sub_invocation = { .command = subcommand };
+  if (!subcommand->fn (argc, argv, &sub_invocation, cancellable, error))
     return FALSE;
 
   return TRUE;

src/ostree/ot-admin-builtin-os-init.c

@@ -38,18 +38,18 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_os_init (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_os_init (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
   gboolean ret = FALSE;
   const char *osname = NULL;
 
-  context = g_option_context_new ("OSNAME - Initialize empty state for given operating system");
+  context = g_option_context_new ("OSNAME");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   if (!ostree_sysroot_ensure_initialized (sysroot, cancellable, error))

src/ostree/ot-admin-builtin-set-origin.c

@@ -45,7 +45,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_set_origin (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_set_origin (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
@@ -60,7 +60,7 @@ ot_admin_builtin_set_origin (int argc, char **argv, GCancellable *cancellable, G
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   if (argc < 3)

src/ostree/ot-admin-builtin-status.c

@@ -37,82 +37,57 @@ static GOptionEntry options[] = {
   { NULL }
 };
 
-static char *
-version_of_commit (OstreeRepo *repo, const char *checksum)
-{
-  g_autoptr(GVariant) variant = NULL;
-  
-  /* Shouldn't fail, but if it does, we ignore it */
-  if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
-                                 &variant, NULL))
-    goto out;
-
-  return ot_admin_checksum_version (variant);
- out:
-  return NULL;
-}
-
 static gboolean
 deployment_get_gpg_verify (OstreeDeployment *deployment,
                            OstreeRepo *repo)
 {
   /* XXX Something like this could be added to the OstreeDeployment
    *     API in libostree if the OstreeRepo parameter is acceptable. */
-
-  GKeyFile *origin;
-  g_autofree char *refspec = NULL;
-  g_autofree char *remote = NULL;
-  gboolean gpg_verify = FALSE;
-
-  origin = ostree_deployment_get_origin (deployment);
+  GKeyFile *origin = ostree_deployment_get_origin (deployment);
 
   if (origin == NULL)
-    goto out;
+    return FALSE;
 
-  refspec = g_key_file_get_string (origin, "origin", "refspec", NULL);
+  g_autofree char *refspec = g_key_file_get_string (origin, "origin", "refspec", NULL);
 
   if (refspec == NULL)
-    goto out;
+    return FALSE;
 
+  g_autofree char *remote = NULL;
   if (!ostree_parse_refspec (refspec, &remote, NULL, NULL))
-    goto out;
+    return FALSE;
 
+  gboolean gpg_verify = FALSE;
   if (remote)
     (void) ostree_repo_remote_get_gpg_verify (repo, remote, &gpg_verify, NULL);
 
-out:
   return gpg_verify;
 }
 
 gboolean
-ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = NULL;
-  g_autoptr(OstreeSysroot) sysroot = NULL;
-  gboolean ret = FALSE;
-  g_autoptr(OstreeRepo) repo = NULL;
-  OstreeDeployment *booted_deployment = NULL;
-  g_autoptr(OstreeDeployment) pending_deployment = NULL;
-  g_autoptr(OstreeDeployment) rollback_deployment = NULL;
-  g_autoptr(GPtrArray) deployments = NULL;
   const int is_tty = isatty (1);
   const char *red_bold_prefix = is_tty ? "\x1b[31m\x1b[1m" : "";
   const char *red_bold_suffix = is_tty ? "\x1b[22m\x1b[0m" : "";
-  guint i;
 
-  context = g_option_context_new ("List deployments");
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
 
+  g_autoptr(OstreeSysroot) sysroot = NULL;
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
-    goto out;
+                                          invocation, &sysroot, cancellable, error))
+    return FALSE;
 
+  g_autoptr(OstreeRepo) repo = NULL;
   if (!ostree_sysroot_get_repo (sysroot, &repo, cancellable, error))
-    goto out;
+    return FALSE;
 
-  deployments = ostree_sysroot_get_deployments (sysroot);
-  booted_deployment = ostree_sysroot_get_booted_deployment (sysroot);
+  g_autoptr(GPtrArray) deployments = ostree_sysroot_get_deployments (sysroot);
+  OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (sysroot);
 
+  g_autoptr(OstreeDeployment) pending_deployment = NULL;
+  g_autoptr(OstreeDeployment) rollback_deployment = NULL;
   if (booted_deployment)
     ostree_sysroot_query_deployments_for (sysroot, NULL, &pending_deployment,
                                           &rollback_deployment);
@@ -123,18 +98,30 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
     }
   else
     {
-      for (i = 0; i < deployments->len; i++)
+      for (guint i = 0; i < deployments->len; i++)
         {
           OstreeDeployment *deployment = deployments->pdata[i];
-          GKeyFile *origin;
           const char *ref = ostree_deployment_get_csum (deployment);
-          OstreeDeploymentUnlockedState unlocked = ostree_deployment_get_unlocked (deployment);
-          g_autofree char *version = version_of_commit (repo, ref);
-          g_autoptr(OstreeGpgVerifyResult) result = NULL;
-          guint jj, n_signatures;
-          GError *local_error = NULL;
 
-          origin = ostree_deployment_get_origin (deployment);
+          /* Load the backing commit; shouldn't normally fail, but if it does,
+           * we stumble on.
+           */
+          g_autoptr(GVariant) commit = NULL;
+          (void)ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, ref,
+                                          &commit, NULL);
+          g_autoptr(GVariant) commit_metadata = NULL;
+          if (commit)
+            commit_metadata = g_variant_get_child_value (commit, 0);
+
+          const char *version = NULL;
+          const char *source_title = NULL;
+          if (commit_metadata)
+            {
+              (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_VERSION, "&s", &version);
+              (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_SOURCE_TITLE, "&s", &source_title);
+            }
+
+          GKeyFile *origin = ostree_deployment_get_origin (deployment);
 
           const char *deployment_status = "";
           if (deployment == pending_deployment)
@@ -149,6 +136,8 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
                    deployment_status);
           if (version)
             g_print ("    Version: %s\n", version);
+
+          OstreeDeploymentUnlockedState unlocked = ostree_deployment_get_unlocked (deployment);
           switch (unlocked)
             {
             case OSTREE_DEPLOYMENT_UNLOCKED_NONE:
@@ -167,6 +156,8 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
                 g_print ("    origin: <unknown origin type>\n");
               else
                 g_print ("    origin refspec: %s\n", origin_refspec);
+              if (source_title)
+                g_print ("    `- %s\n", source_title);
             }
 
           if (deployment_get_gpg_verify (deployment, repo))
@@ -174,8 +165,10 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
               g_autoptr(GString) output_buffer = g_string_sized_new (256);
               /* Print any digital signatures on this commit. */
 
-              result = ostree_repo_verify_commit_ext (repo, ref, NULL, NULL,
-                                                      cancellable, &local_error);
+              g_autoptr(GError) local_error = NULL;
+              g_autoptr(OstreeGpgVerifyResult) result =
+                ostree_repo_verify_commit_ext (repo, ref, NULL, NULL,
+                                               cancellable, &local_error);
 
               /* G_IO_ERROR_NOT_FOUND just means the commit is not signed. */
               if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
@@ -185,13 +178,12 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
                 }
               else if (local_error != NULL)
                 {
-                  g_propagate_error (error, local_error);
-                  goto out;
+                  g_propagate_error (error, g_steal_pointer (&local_error));
+                  return FALSE;
                 }
 
-              n_signatures = ostree_gpg_verify_result_count_all (result);
-
-              for (jj = 0; jj < n_signatures; jj++)
+              const guint n_signatures = ostree_gpg_verify_result_count_all (result);
+              for (guint jj = 0; jj < n_signatures; jj++)
                 {
                   ostree_gpg_verify_result_describe (result, jj, output_buffer, "    GPG: ",
                                                      OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT);
@@ -202,7 +194,5 @@ ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GErro
         }
     }
 
-  ret = TRUE;
- out:
-  return ret;
+  return TRUE;
 }

src/ostree/ot-admin-builtin-switch.c

@@ -44,14 +44,14 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_switch (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context =
-    g_option_context_new ("REF - Construct new tree from REF and deploy it");
+    g_option_context_new ("REF");
   g_autoptr(OstreeSysroot) sysroot = NULL;
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     return FALSE;
 
   if (argc < 2)

src/ostree/ot-admin-builtin-undeploy.c

@@ -37,7 +37,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_undeploy (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
@@ -46,11 +46,11 @@ ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GEr
   g_autoptr(GPtrArray) current_deployments = NULL;
   g_autoptr(OstreeDeployment) target_deployment = NULL;
 
-  context = g_option_context_new ("INDEX - Delete deployment INDEX");
+  context = g_option_context_new ("INDEX");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     return FALSE;
 
   if (argc < 2)

src/ostree/ot-admin-builtin-unlock.c

@@ -43,7 +43,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_unlock (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_unlock (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
@@ -51,11 +51,11 @@ ot_admin_builtin_unlock (int argc, char **argv, GCancellable *cancellable, GErro
   OstreeDeployment *booted_deployment = NULL;
   OstreeDeploymentUnlockedState target_state;
 
-  context = g_option_context_new ("Make the current deployment mutable (as a hotfix or development)");
+  context = g_option_context_new ("");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
   
   if (argc > 1)

src/ostree/ot-admin-builtin-upgrade.c

@@ -54,14 +54,14 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_builtin_upgrade (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = g_option_context_new ("Construct new tree from current origin and deploy it, if it changed");
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
 
   g_autoptr(OstreeSysroot) sysroot = NULL;
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     return FALSE;
 
   if (opt_pull_only && opt_deploy_only)

src/ostree/ot-admin-builtins.h

@@ -25,19 +25,25 @@
 
 G_BEGIN_DECLS
 
-gboolean ot_admin_builtin_selinux_ensure_labeled (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_os_init (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_install (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_instutil (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_init_fs (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_undeploy (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_deploy (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_cleanup (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_unlock (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_status (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_set_origin (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_switch (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_builtin_upgrade (int argc, char **argv, GCancellable *cancellable, GError **error);
+#define BUILTINPROTO(name) gboolean ot_admin_builtin_ ## name (int argc, char **argv, \
+                                                               OstreeCommandInvocation *invocation, \
+                                                               GCancellable *cancellable, GError **error)
+
+BUILTINPROTO(selinux_ensure_labeled);
+BUILTINPROTO(os_init);
+BUILTINPROTO(install);
+BUILTINPROTO(instutil);
+BUILTINPROTO(init_fs);
+BUILTINPROTO(undeploy);
+BUILTINPROTO(deploy);
+BUILTINPROTO(cleanup);
+BUILTINPROTO(unlock);
+BUILTINPROTO(status);
+BUILTINPROTO(set_origin);
+BUILTINPROTO(diff);
+BUILTINPROTO(switch);
+BUILTINPROTO(upgrade);
+
+#undef BUILTINPROTO
 
 G_END_DECLS

src/ostree/ot-admin-instutil-builtin-grub2-generate.c

@@ -38,18 +38,18 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_instutil_builtin_grub2_generate (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_instutil_builtin_grub2_generate (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   guint bootversion;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
 
-  context = g_option_context_new ("[BOOTVERSION] - generate GRUB2 configuration from given BLS entries");
+  context = g_option_context_new ("[BOOTVERSION]");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   if (argc >= 2)

src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c

@@ -183,7 +183,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_instutil_builtin_selinux_ensure_labeled (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_instutil_builtin_selinux_ensure_labeled (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   const char *policy_name;
@@ -196,11 +196,11 @@ ot_admin_instutil_builtin_selinux_ensure_labeled (int argc, char **argv, GCancel
   g_autoptr(OstreeSysroot) sysroot = NULL;
   g_autoptr(GFile) deployment_path = NULL;
 
-  context = g_option_context_new ("[SUBPATH PREFIX] - relabel all or part of a deployment");
+  context = g_option_context_new ("[SUBPATH PREFIX]");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   deployments = ostree_sysroot_get_deployments (sysroot);

src/ostree/ot-admin-instutil-builtin-set-kargs.c

@@ -48,7 +48,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ot_admin_instutil_builtin_set_kargs (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_admin_instutil_builtin_set_kargs (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   guint i;
@@ -56,13 +56,13 @@ ot_admin_instutil_builtin_set_kargs (int argc, char **argv, GCancellable *cancel
   OstreeDeployment *first_deployment = NULL;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeSysroot) sysroot = NULL;
-  __attribute__((cleanup(_ostree_kernel_args_cleanup))) OstreeKernelArgs *kargs = NULL;
+  g_autoptr(OstreeKernelArgs) kargs = NULL;
 
-  context = g_option_context_new ("ARGS - set new kernel command line arguments");
+  context = g_option_context_new ("ARGS");
 
   if (!ostree_admin_option_context_parse (context, options, &argc, &argv,
                                           OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER | OSTREE_ADMIN_BUILTIN_FLAG_UNLOCKED,
-                                          &sysroot, cancellable, error))
+                                          invocation, &sysroot, cancellable, error))
     goto out;
 
   deployments = ostree_sysroot_get_deployments (sysroot);

src/ostree/ot-admin-instutil-builtins.h

@@ -23,8 +23,8 @@
 
 G_BEGIN_DECLS
 
-gboolean ot_admin_instutil_builtin_selinux_ensure_labeled (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_instutil_builtin_set_kargs (int argc, char **argv, GCancellable *cancellable, GError **error);
-gboolean ot_admin_instutil_builtin_grub2_generate (int argc, char **argv, GCancellable *cancellable, GError **error);
+gboolean ot_admin_instutil_builtin_selinux_ensure_labeled (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
+gboolean ot_admin_instutil_builtin_set_kargs (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
+gboolean ot_admin_instutil_builtin_grub2_generate (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
 
 G_END_DECLS

src/ostree/ot-builtin-admin.c

@@ -30,38 +30,59 @@
 
 #include <glib/gi18n.h>
 
-typedef struct {
-  const char *name;
-  gboolean (*fn) (int argc, char **argv, GCancellable *cancellable, GError **error);
-} OstreeAdminCommand;
-
-static OstreeAdminCommand admin_subcommands[] = {
-  { "cleanup", ot_admin_builtin_cleanup },
-  { "config-diff", ot_admin_builtin_diff },
-  { "deploy", ot_admin_builtin_deploy }, 
-  { "init-fs", ot_admin_builtin_init_fs },
-  { "instutil", ot_admin_builtin_instutil },
-  { "os-init", ot_admin_builtin_os_init },
-  { "set-origin", ot_admin_builtin_set_origin },
-  { "status", ot_admin_builtin_status },
-  { "switch", ot_admin_builtin_switch },
-  { "undeploy", ot_admin_builtin_undeploy },
-  { "unlock", ot_admin_builtin_unlock }, 
-  { "upgrade", ot_admin_builtin_upgrade },
-  { NULL, NULL }
+static OstreeCommand admin_subcommands[] = {
+  { "cleanup", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_cleanup,
+    "Delete untagged deployments and repository objects" },
+  { "config-diff", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_diff,
+    "Diff current /etc configuration versus default" },
+  { "deploy", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_deploy,
+    "Checkout revision REFSPEC as the new default deployment" },
+  { "init-fs", OSTREE_BUILTIN_FLAG_NO_REPO,
+     ot_admin_builtin_init_fs,
+    "Initialize a root filesystem" },
+  { "instutil", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_instutil,
+    "Provide instutil commands, allow admin to change boot configuration and relabel selinux " },
+  { "os-init", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_os_init,
+    "Initialize empty state for given operating system" },
+  { "set-origin", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_set_origin,
+    "Set Origin and create a new origin file" },
+  { "status", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_status,
+    "List deployments" },
+  { "switch", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_switch,
+    "Construct new tree from REF and deploy it" },
+  { "undeploy", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_undeploy,
+    "Delete deployment INDEX" },
+  { "unlock", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_unlock,
+    "Make the current deployment mutable (as a hotfix or development)" },
+  { "upgrade", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_admin_builtin_upgrade,
+    "Construct new tree from current origin and deploy it, if it changed" },
+  { NULL, 0, NULL, NULL }
 };
 
 static GOptionContext *
 ostree_admin_option_context_new_with_commands (void)
 {
-  OstreeAdminCommand *command = admin_subcommands;
+  OstreeCommand *command = admin_subcommands;
   GOptionContext *context = g_option_context_new ("--print-current-dir|COMMAND");
 
   g_autoptr(GString) summary = g_string_new ("Builtin \"admin\" Commands:");
 
   while (command->name != NULL)
     {
-      g_string_append_printf (summary, "\n  %s", command->name);
+      g_string_append_printf (summary, "\n  %-19s", command->name);
+        if (command->description != NULL)
+          g_string_append_printf (summary, "%s", command->description);
       command++;
     }
 
@@ -71,11 +92,11 @@ ostree_admin_option_context_new_with_commands (void)
 }
 
 gboolean
-ostree_builtin_admin (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_admin (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   const char *subcommand_name = NULL;
-  OstreeAdminCommand *subcommand;
+  OstreeCommand *subcommand;
   g_autofree char *prgname = NULL;
   int in, out;
 
@@ -126,7 +147,7 @@ ostree_builtin_admin (int argc, char **argv, GCancellable *cancellable, GError *
       /* This will not return for some options (e.g. --version). */
       if (ostree_admin_option_context_parse (context, NULL, &argc, &argv,
                                              OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT,
-                                             NULL, cancellable, error))
+                                             invocation, NULL, cancellable, error))
         {
           if (subcommand_name == NULL)
             {
@@ -149,7 +170,8 @@ ostree_builtin_admin (int argc, char **argv, GCancellable *cancellable, GError *
   prgname = g_strdup_printf ("%s %s", g_get_prgname (), subcommand_name);
   g_set_prgname (prgname);
 
-  if (!subcommand->fn (argc, argv, cancellable, error))
+  OstreeCommandInvocation sub_invocation = { .command = subcommand };
+  if (!subcommand->fn (argc, argv, &sub_invocation, cancellable, error))
     goto out;
  
   ret = TRUE;

src/ostree/ot-builtin-cat.c

@@ -55,11 +55,11 @@ cat_one_file (GFile         *f,
 }
 
 gboolean
-ostree_builtin_cat (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_cat (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = g_option_context_new ("COMMIT PATH... - Concatenate contents of files");
+  g_autoptr(GOptionContext) context = g_option_context_new ("COMMIT PATH...");
   g_autoptr(OstreeRepo) repo = NULL;
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (argc <= 2)

src/ostree/ot-builtin-checkout.c

@@ -279,7 +279,7 @@ process_many_checkouts (OstreeRepo         *repo,
 }
 
 gboolean
-ostree_builtin_checkout (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_checkout (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -288,9 +288,9 @@ ostree_builtin_checkout (int argc, char **argv, GCancellable *cancellable, GErro
   const char *destination;
   g_autofree char *resolved_commit = NULL;
 
-  context = g_option_context_new ("COMMIT [DESTINATION] - Check out a commit into a filesystem tree");
+  context = g_option_context_new ("COMMIT [DESTINATION]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (opt_disable_fsync)

src/ostree/ot-builtin-checksum.c

@@ -32,12 +32,16 @@
  * man page (man/ostree-checksum.xml) when changing the option list.
  */
 
+static gboolean opt_ignore_xattrs;
+
 static GOptionEntry options[] = {
+  { "ignore-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_ignore_xattrs, "Don't include xattrs in checksum", NULL },
   { NULL }
 };
 
 typedef struct {
   GError **error;
+  gboolean success;
   GMainLoop *loop;
 } AsyncChecksumData;
 
@@ -46,50 +50,56 @@ on_checksum_received (GObject    *obj,
                       GAsyncResult  *result,
                       gpointer       user_data)
 {
-  g_autofree guchar *csum = NULL;
-  g_autofree char *checksum = NULL;
   AsyncChecksumData *data = user_data;
 
-  if (ostree_checksum_file_async_finish ((GFile*)obj, result, &csum, data->error))
+  g_autofree guchar *csum_bytes = NULL;
+  data->success =
+    ostree_checksum_file_async_finish ((GFile*)obj, result, &csum_bytes, data->error);
+  if (data->success)
     {
-      checksum = ostree_checksum_from_bytes (csum);
-      g_print ("%s\n", checksum);
+      char csum[OSTREE_SHA256_STRING_LEN+1];
+      ostree_checksum_inplace_from_bytes (csum_bytes, csum);
+      g_print ("%s\n", csum);
     }
-  
+
   g_main_loop_quit (data->loop);
 }
 
 gboolean
-ostree_builtin_checksum (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_checksum (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = NULL;
-  gboolean ret = FALSE;
-  g_autoptr(GFile) f = NULL;
-  AsyncChecksumData data = { 0, };
+  g_autoptr(GOptionContext) context =
+    g_option_context_new ("PATH");
+  if (!ostree_option_context_parse (context, options, &argc, &argv,
+                                    invocation, NULL, cancellable, error))
+    return FALSE;
 
-  context = g_option_context_new ("PATH - Checksum a file or directory");
+  if (argc < 2)
+    return glnx_throw (error, "A filename must be given");
+  const char *path = argv[1];
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, error))
-    goto out;
-
-  if (argc > 1)
-    f = g_file_new_for_path (argv[1]);
-  else
+  /* for test coverage, use the async API if no flags are needed */
+  if (!opt_ignore_xattrs)
     {
-      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "A filename must be given");
-      goto out;
+      g_autoptr(GFile) f = g_file_new_for_path (path);
+      g_autoptr(GMainLoop) loop = g_main_loop_new (NULL, FALSE);
+
+      AsyncChecksumData data = { 0, };
+
+      data.loop = loop;
+      data.error = error;
+      ostree_checksum_file_async (f, OSTREE_OBJECT_TYPE_FILE, G_PRIORITY_DEFAULT,
+                                  cancellable, on_checksum_received, &data);
+      g_main_loop_run (data.loop);
+      return data.success;
     }
 
-  data.loop = g_main_loop_new (NULL, FALSE);
-  data.error = error;
-  ostree_checksum_file_async (f, OSTREE_OBJECT_TYPE_FILE, G_PRIORITY_DEFAULT, cancellable, on_checksum_received, &data);
-  
-  g_main_loop_run (data.loop);
+  g_autofree char *checksum = NULL;
+  if (!ostree_checksum_file_at (AT_FDCWD, path, NULL, OSTREE_OBJECT_TYPE_FILE,
+                                OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS, &checksum,
+                                cancellable, error))
+    return FALSE;
 
-  ret = TRUE;
- out:
-  if (data.loop)
-    g_main_loop_unref (data.loop);
-  return ret;
+  g_print ("%s\n", checksum);
+  return TRUE;
 }

src/ostree/ot-builtin-commit.c

@@ -50,6 +50,7 @@ static char *opt_tar_pathname_filter;
 static gboolean opt_no_xattrs;
 static char *opt_selinux_policy;
 static gboolean opt_canonical_permissions;
+static gboolean opt_consume;
 static char **opt_trees;
 static gint opt_owner_uid = -1;
 static gint opt_owner_gid = -1;
@@ -102,6 +103,7 @@ static GOptionEntry options[] = {
   { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL },
   { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" },
   { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
+  { "consume", 0, 0, G_OPTION_ARG_NONE, &opt_consume, "Consume (delete) content after commit (for local directories)", NULL },
   { "table-output", 0, 0, G_OPTION_ARG_NONE, &opt_table_output, "Output more information in a KEY: VALUE format", NULL },
   { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"},
   { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"},
@@ -405,7 +407,7 @@ fill_bindings (OstreeRepo    *repo,
 }
 
 gboolean
-ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -429,9 +431,9 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
   struct CommitFilterData filter_data = { 0, };
   g_autofree char *commit_body = NULL;
 
-  context = g_option_context_new ("[PATH] - Commit a new revision");
+  context = g_option_context_new ("[PATH]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))
@@ -476,6 +478,8 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
 
   if (opt_no_xattrs)
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS;
+  if (opt_consume)
+    flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME;
   if (opt_canonical_permissions)
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS;
   if (opt_generate_sizes)
@@ -497,7 +501,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
                                                   &filter_data, NULL);
       if (opt_selinux_policy)
         {
-          glnx_fd_close int rootfs_dfd = -1;
+          glnx_autofd int rootfs_dfd = -1;
           if (!glnx_opendirat (AT_FDCWD, opt_selinux_policy, TRUE, &rootfs_dfd, error))
             {
               g_prefix_error (error, "selinux-policy: ");

src/ostree/ot-builtin-config.c

@@ -56,7 +56,7 @@ split_key_string (const char   *k,
 }
 
 gboolean
-ostree_builtin_config (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_config (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -68,9 +68,9 @@ ostree_builtin_config (int argc, char **argv, GCancellable *cancellable, GError
   g_autofree char *key = NULL;
   GKeyFile *config = NULL;
 
-  context = g_option_context_new ("(get KEY|set KEY VALUE) - Change repo configuration settings");
+  context = g_option_context_new ("(get KEY|set KEY VALUE)");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 2)

src/ostree/ot-builtin-create-usb.c

@@ -43,6 +43,7 @@ static GOptionEntry options[] =
 gboolean
 ostree_builtin_create_usb (int            argc,
                            char         **argv,
+                           OstreeCommandInvocation *invocation,
                            GCancellable  *cancellable,
                            GError       **error)
 {
@@ -50,12 +51,12 @@ ostree_builtin_create_usb (int            argc,
   g_autoptr(OstreeAsyncProgress) progress = NULL;
   g_auto(GLnxConsoleRef) console = { 0, };
 
-  context = g_option_context_new ("MOUNT-PATH COLLECTION-ID REF [COLLECTION-ID REF...] - Copy the refs to a USB stick");
+  context = g_option_context_new ("MOUNT-PATH COLLECTION-ID REF [COLLECTION-ID REF...]");
 
   /* Parse options. */
   g_autoptr(OstreeRepo) src_repo = NULL;
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &src_repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &src_repo, cancellable, error))
     return FALSE;
 
   if (argc < 2)
@@ -80,7 +81,7 @@ ostree_builtin_create_usb (int            argc,
   const char *mount_root_path = argv[1];
   struct stat mount_root_stbuf;
 
-  glnx_fd_close int mount_root_dfd = -1;
+  glnx_autofd int mount_root_dfd = -1;
   if (!glnx_opendirat (AT_FDCWD, mount_root_path, TRUE, &mount_root_dfd, error))
     return FALSE;
   if (!glnx_fstat (mount_root_dfd, &mount_root_stbuf, error))
@@ -113,7 +114,7 @@ ostree_builtin_create_usb (int            argc,
   OstreeRepoMode mode = OSTREE_REPO_MODE_BARE_USER;
 
   if (TEMP_FAILURE_RETRY (fgetxattr (mount_root_dfd, "user.test", NULL, 0)) < 0 &&
-      errno == ENOTSUP)
+      (errno == ENOTSUP || errno == EOPNOTSUPP))
     mode = OSTREE_REPO_MODE_ARCHIVE;
 
   g_debug ("%s: Creating repository in mode %u", G_STRFUNC, mode);

src/ostree/ot-builtin-diff.c

@@ -126,7 +126,7 @@ object_set_total_size (OstreeRepo    *repo,
 }
 
 gboolean
-ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_diff (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
@@ -140,9 +140,9 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError **
   g_autoptr(GPtrArray) removed = NULL;
   g_autoptr(GPtrArray) added = NULL;
 
-  context = g_option_context_new ("REV TARGETDIR - Compare directory TARGETDIR against revision REV");
+  context = g_option_context_new ("REV TARGETDIR");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 2)

src/ostree/ot-builtin-export.c

@@ -62,7 +62,7 @@ propagate_libarchive_error (GError      **error,
 #endif
 
 gboolean
-ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_export (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -77,9 +77,9 @@ ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError
   OstreeRepoExportArchiveOptions opts = { 0, };
 #endif
 
-  context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format");
+  context = g_option_context_new ("COMMIT");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
 #ifdef HAVE_LIBARCHIVE  

src/ostree/ot-builtin-find-remotes.c

@@ -120,6 +120,7 @@ collection_ref_free0 (OstreeCollectionRef *ref)
 gboolean
 ostree_builtin_find_remotes (int            argc,
                              char         **argv,
+                             OstreeCommandInvocation *invocation,
                              GCancellable  *cancellable,
                              GError       **error)
 {
@@ -133,10 +134,10 @@ ostree_builtin_find_remotes (int            argc,
   g_auto(GLnxConsoleRef) console = { 0, };
   g_autoptr(GHashTable) refs_found = NULL;  /* set (element-type OstreeCollectionRef) */
 
-  context = g_option_context_new ("COLLECTION-ID REF [COLLECTION-ID REF...] - Find remotes to serve the given refs");
+  context = g_option_context_new ("COLLECTION-ID REF [COLLECTION-ID REF...]");
 
   /* Parse options. */
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (!ostree_ensure_repo_writable (repo, error))

src/ostree/ot-builtin-fsck.c

@@ -214,13 +214,13 @@ fsck_reachable_objects_from_commits (OstreeRepo            *repo,
 }
 
 gboolean
-ostree_builtin_fsck (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(OstreeRepo) repo = NULL;
   gboolean found_corruption = FALSE;
 
-  g_autoptr(GOptionContext) context = g_option_context_new ("- Check the repository for consistency");
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (!opt_quiet)

src/ostree/ot-builtin-gpg-sign.c

@@ -200,7 +200,7 @@ out:
 }
 
 gboolean
-ostree_builtin_gpg_sign (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_gpg_sign (int argc, char **argv,OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -210,9 +210,9 @@ ostree_builtin_gpg_sign (int argc, char **argv, GCancellable *cancellable, GErro
   int n_key_ids, ii;
   gboolean ret = FALSE;
 
-  context = g_option_context_new ("COMMIT KEY-ID... - Sign a commit");
+  context = g_option_context_new ("COMMIT KEY-ID...");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 2)

src/ostree/ot-builtin-init.c

@@ -45,16 +45,16 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ostree_builtin_init (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_init (int argc, char **argv,OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
   gboolean ret = FALSE;
   OstreeRepoMode mode;
 
-  context = g_option_context_new ("- Initialize a new empty repository");
+  context = g_option_context_new ("");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NO_CHECK, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_repo_mode_from_string (opt_mode, &mode, error))

src/ostree/ot-builtin-log.c

@@ -82,6 +82,7 @@ out:
 gboolean
 ostree_builtin_log (int           argc,
                     char        **argv,
+                    OstreeCommandInvocation *invocation,
                     GCancellable *cancellable,
                     GError      **error)
 {
@@ -92,9 +93,9 @@ ostree_builtin_log (int           argc,
   g_autofree char *checksum = NULL;
   OstreeDumpFlags flags = OSTREE_DUMP_NONE;
 
-  context = g_option_context_new ("REF - Show log starting at commit or ref");
+  context = g_option_context_new ("REF");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (opt_raw)

src/ostree/ot-builtin-ls.c

@@ -240,7 +240,7 @@ print_one_argument (OstreeRepo   *repo,
 }
 
 gboolean
-ostree_builtin_ls (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_ls (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -249,9 +249,9 @@ ostree_builtin_ls (int argc, char **argv, GCancellable *cancellable, GError **er
   int i;
   g_autoptr(GFile) root = NULL;
 
-  context = g_option_context_new ("COMMIT [PATH...] - List file paths");
+  context = g_option_context_new ("COMMIT [PATH...]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc <= 1)

src/ostree/ot-builtin-prune.c

@@ -144,11 +144,11 @@ traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
 }
 
 gboolean
-ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = g_option_context_new ("- Search for unreachable objects");
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
   g_autoptr(OstreeRepo) repo = NULL;
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (!opt_no_prune && !ostree_ensure_repo_writable (repo, error))

src/ostree/ot-builtin-pull-local.c

@@ -64,7 +64,7 @@ noninteractive_console_progress_changed (OstreeAsyncProgress *progress,
 }
 
 gboolean
-ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
@@ -76,9 +76,9 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr
   g_autoptr(GPtrArray) refs_to_fetch = NULL;
   OstreeRepoPullFlags pullflags = 0;
 
-  context = g_option_context_new ("SRC_REPO [REFS...] -  Copy data from SRC_REPO");
+  context = g_option_context_new ("SRC_REPO [REFS...]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))

src/ostree/ot-builtin-pull.c

@@ -143,7 +143,7 @@ noninteractive_console_progress_changed (OstreeAsyncProgress *progress,
 }
 
 gboolean
-ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -155,9 +155,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
   g_autoptr(OstreeAsyncProgress) progress = NULL;
   gulong signal_handler_id = 0;
 
-  context = g_option_context_new ("REMOTE [BRANCH...] - Download data from remote repository");
+  context = g_option_context_new ("REMOTE [BRANCH...]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))

src/ostree/ot-builtin-refs.c

@@ -261,16 +261,16 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab
 }
 
 gboolean
-ostree_builtin_refs (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_refs (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
   int i;
 
-  context = g_option_context_new ("[PREFIX] - List refs");
+  context = g_option_context_new ("[PREFIX]");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc >= 2)

src/ostree/ot-builtin-remote.c

@@ -25,38 +25,56 @@
 #include "ot-builtins.h"
 #include "ot-remote-builtins.h"
 
-typedef struct {
-  const char *name;
-  gboolean (*fn) (int argc, char **argv, GCancellable *cancellable, GError **error);
-} OstreeRemoteCommand;
-
-static OstreeRemoteCommand remote_subcommands[] = {
-  { "add", ot_remote_builtin_add },
-  { "delete", ot_remote_builtin_delete },
-  { "show-url", ot_remote_builtin_show_url },
-  { "list", ot_remote_builtin_list },
-  { "gpg-import", ot_remote_builtin_gpg_import },
+static OstreeCommand remote_subcommands[] = {
+  { "add", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_remote_builtin_add,
+    "Add a remote repository" },
+  { "delete", OSTREE_BUILTIN_FLAG_NO_REPO,
+    ot_remote_builtin_delete,
+    "Delete a remote repository" },
+  { "show-url", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_show_url,
+    "Show remote repository URL" },
+  { "list", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_list,
+    "List remote repository names" },
+  { "gpg-import", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_gpg_import,
+    "Import GPG keys" },
 #ifdef HAVE_LIBSOUP
-  { "add-cookie", ot_remote_builtin_add_cookie },
-  { "delete-cookie", ot_remote_builtin_delete_cookie },
-  { "list-cookies", ot_remote_builtin_list_cookies },
+  { "add-cookie", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_add_cookie,
+    "Add a cookie to remote" },
+  { "delete-cookie", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_delete_cookie,
+    "Remove one cookie from remote" },
+  { "list-cookies", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_list_cookies,
+    "Show remote repository cookies" },
 #endif
-  { "refs", ot_remote_builtin_refs },
-  { "summary", ot_remote_builtin_summary },
-  { NULL, NULL }
+  { "refs", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_refs,
+    "List remote refs" },
+  { "summary", OSTREE_BUILTIN_FLAG_NONE,
+    ot_remote_builtin_summary,
+    "Show remote summary" },
+  { NULL, 0, NULL, NULL }
 };
 
 static GOptionContext *
 remote_option_context_new_with_commands (void)
 {
-  OstreeRemoteCommand *subcommand = remote_subcommands;
+  OstreeCommand *subcommand = remote_subcommands;
   GOptionContext *context = g_option_context_new ("COMMAND");
 
   g_autoptr(GString) summary = g_string_new ("Builtin \"remote\" Commands:");
 
   while (subcommand->name != NULL)
     {
-      g_string_append_printf (summary, "\n  %s", subcommand->name);
+      g_string_append_printf (summary, "\n  %-18s", subcommand->name);
+      if (subcommand->description != NULL)
+        g_string_append_printf (summary, "%s", subcommand->description);
+
       subcommand++;
     }
 
@@ -66,9 +84,9 @@ remote_option_context_new_with_commands (void)
 }
 
 gboolean
-ostree_builtin_remote (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_remote (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  OstreeRemoteCommand *subcommand;
+  OstreeCommand *subcommand;
   const char *subcommand_name = NULL;
   g_autofree char *prgname = NULL;
   gboolean ret = FALSE;
@@ -114,7 +132,7 @@ ostree_builtin_remote (int argc, char **argv, GCancellable *cancellable, GError
 
       /* This will not return for some options (e.g. --version). */
       if (ostree_option_context_parse (context, NULL, &argc, &argv,
-                                       OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable,
+                                       invocation, NULL, cancellable,
                                        error))
         {
           if (subcommand_name == NULL)
@@ -138,7 +156,8 @@ ostree_builtin_remote (int argc, char **argv, GCancellable *cancellable, GError
   prgname = g_strdup_printf ("%s %s", g_get_prgname (), subcommand_name);
   g_set_prgname (prgname);
 
-  if (!subcommand->fn (argc, argv, cancellable, error))
+  OstreeCommandInvocation sub_invocation = { .command = subcommand };
+  if (!subcommand->fn (argc, argv, &sub_invocation, cancellable, error))
     goto out;
 
   ret = TRUE;

src/ostree/ot-builtin-reset.c

@@ -38,6 +38,7 @@ static GOptionEntry options[] = {
 gboolean
 ostree_builtin_reset (int           argc,
                       char        **argv,
+                      OstreeCommandInvocation *invocation,
                       GCancellable *cancellable,
                       GError      **error)
 {
@@ -50,9 +51,9 @@ ostree_builtin_reset (int           argc,
   g_autofree char *checksum = NULL;
 
   /* FIXME: Add support for collection–refs. */
-  context = g_option_context_new ("REF COMMIT - Reset a REF to a previous COMMIT");
+  context = g_option_context_new ("REF COMMIT");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))

src/ostree/ot-builtin-rev-parse.c

@@ -36,7 +36,7 @@ static GOptionEntry options[] = {
 };
 
 gboolean
-ostree_builtin_rev_parse (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_rev_parse (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
@@ -45,9 +45,9 @@ ostree_builtin_rev_parse (int argc, char **argv, GCancellable *cancellable, GErr
   int i;
   g_autofree char *resolved_rev = NULL;
 
-  context = g_option_context_new ("REV - Output the target of a rev");
+  context = g_option_context_new ("REV");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 2)

src/ostree/ot-builtin-show.c

@@ -58,7 +58,10 @@ do_print_variant_generic (const GVariantType *type,
 {
   g_autoptr(GVariant) variant = NULL;
 
-  if (!ot_util_variant_map_at (AT_FDCWD, filename, type, TRUE, &variant, error))
+  glnx_autofd int fd = -1;
+  if (!glnx_openat_rdonly (AT_FDCWD, filename, TRUE, &fd, error))
+    return FALSE;
+  if (!ot_variant_read_fd (fd, 0, type, FALSE, &variant, error))
     return FALSE;
 
   ot_dump_variant (variant);
@@ -223,12 +226,12 @@ print_if_found (OstreeRepo        *repo,
 }
 
 gboolean
-ostree_builtin_show (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GOptionContext) context = g_option_context_new ("OBJECT - Output a metadata object");
+  g_autoptr(GOptionContext) context = g_option_context_new ("OBJECT");
 
   g_autoptr(OstreeRepo) repo = NULL;
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (argc <= 1)

src/ostree/ot-builtin-static-delta.c

@@ -39,7 +39,7 @@ static gboolean opt_inline;
 static gboolean opt_disable_bsdiff;
 static gboolean opt_if_not_exists;
 
-#define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error)
+#define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 
 BUILTINPROTO(list);
 BUILTINPROTO(show);
@@ -50,12 +50,22 @@ BUILTINPROTO(apply_offline);
 #undef BUILTINPROTO
 
 static OstreeCommand static_delta_subcommands[] = {
-  { "list", ot_static_delta_builtin_list },
-  { "show", ot_static_delta_builtin_show },
-  { "delete", ot_static_delta_builtin_delete },
-  { "generate", ot_static_delta_builtin_generate },
-  { "apply-offline", ot_static_delta_builtin_apply_offline },
-  { NULL, NULL }
+  { "list", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_list,
+    "List static delta files" },
+  { "show", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_show,
+    "Dump information on a delta" },
+  { "delete", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_delete,
+    "Remove a delta" },
+  { "generate", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_generate,
+    "Generate static delta files" },
+  { "apply-offline", OSTREE_BUILTIN_FLAG_NONE,
+    ot_static_delta_builtin_apply_offline,
+    "Apply static delta file" },
+  { NULL, 0, NULL, NULL }
 };
 
 /* ATTENTION:
@@ -105,7 +115,7 @@ static_delta_usage (char    **argv,
 
   while (command->name)
     {
-      print_func ("  %s\n", command->name);
+      print_func ("  %-17s%s\n", command->name, command->description ?: "");
       command++;
     }
 
@@ -113,12 +123,12 @@ static_delta_usage (char    **argv,
 }
 
 static gboolean
-ot_static_delta_builtin_list (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(OstreeRepo) repo = NULL;
-  g_autoptr(GOptionContext) context = g_option_context_new ("- list static delta files");
+  g_autoptr(GOptionContext) context = g_option_context_new ("");
   if (!ostree_option_context_parse (context, list_options, &argc, &argv,
-                                    OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+                                    invocation, &repo, cancellable, error))
     return FALSE;
 
   g_autoptr(GPtrArray) delta_names = NULL;
@@ -137,16 +147,16 @@ ot_static_delta_builtin_list (int argc, char **argv, GCancellable *cancellable,
 }
 
 static gboolean
-ot_static_delta_builtin_show (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
   const char *delta_id = NULL;
 
-  context = g_option_context_new ("- Dump information on a delta");
+  context = g_option_context_new ("");
 
-  if (!ostree_option_context_parse (context, list_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, list_options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 3)
@@ -167,16 +177,16 @@ ot_static_delta_builtin_show (int argc, char **argv, GCancellable *cancellable,
 }
 
 static gboolean
-ot_static_delta_builtin_delete (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_static_delta_builtin_delete (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
   const char *delta_id = NULL;
 
-  context = g_option_context_new ("- Remove a delta");
+  context = g_option_context_new ("");
 
-  if (!ostree_option_context_parse (context, list_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, list_options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (argc < 3)
@@ -198,14 +208,14 @@ ot_static_delta_builtin_delete (int argc, char **argv, GCancellable *cancellable
 
 
 static gboolean
-ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_static_delta_builtin_generate (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
 
-  context = g_option_context_new ("[TO] - Generate static delta files");
-  if (!ostree_option_context_parse (context, generate_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  context = g_option_context_new ("[TO]");
+  if (!ostree_option_context_parse (context, generate_options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))
@@ -347,7 +357,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
 }
 
 static gboolean
-ot_static_delta_builtin_apply_offline (int argc, char **argv, GCancellable *cancellable, GError **error)
+ot_static_delta_builtin_apply_offline (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean ret = FALSE;
   const char *patharg;
@@ -355,8 +365,8 @@ ot_static_delta_builtin_apply_offline (int argc, char **argv, GCancellable *canc
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
 
-  context = g_option_context_new ("- Apply static delta file");
-  if (!ostree_option_context_parse (context, apply_offline_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  context = g_option_context_new ("");
+  if (!ostree_option_context_parse (context, apply_offline_options, &argc, &argv, invocation, &repo, cancellable, error))
     goto out;
 
   if (!ostree_ensure_repo_writable (repo, error))
@@ -387,7 +397,7 @@ ot_static_delta_builtin_apply_offline (int argc, char **argv, GCancellable *canc
 }
 
 gboolean
-ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_static_delta (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   gboolean want_help = FALSE;
   const char *cmdname = NULL;
@@ -438,5 +448,6 @@ ostree_builtin_static_delta (int argc, char **argv, GCancellable *cancellable, G
   g_autofree char *prgname = g_strdup_printf ("%s %s", g_get_prgname (), cmdname);
   g_set_prgname (prgname);
 
-  return command->fn (argc, argv, cancellable, error);
+  OstreeCommandInvocation sub_invocation = { .command = command };
+  return command->fn (argc, argv, &sub_invocation, cancellable, error);
 }

src/ostree/ot-builtin-summary.c

@@ -81,15 +81,15 @@ build_additional_metadata (const char * const  *args,
 }
 
 gboolean
-ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GOptionContext) context = NULL;
   g_autoptr(OstreeRepo) repo = NULL;
   OstreeDumpFlags flags = OSTREE_DUMP_NONE;
 
-  context = g_option_context_new ("Manage summary metadata");
+  context = g_option_context_new ("");
 
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error))
+  if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error))
     return FALSE;
 
   if (opt_update)
@@ -217,7 +217,10 @@ ostree_builtin_summary (int argc, char **argv, GCancellable *cancellable, GError
       if (opt_raw)
         flags |= OSTREE_DUMP_RAW;
 
-      summary_data = ot_file_mapat_bytes (repo->repo_dir_fd, "summary", error);
+      glnx_autofd int fd = -1;
+      if (!glnx_openat_rdonly (repo->repo_dir_fd, "summary", TRUE, &fd, error))
+        return FALSE;
+      summary_data = ot_fd_readall_or_mmap (fd, 0, error);
       if (!summary_data)
         return FALSE;
 

src/ostree/ot-builtin-trivial-httpd.c

@@ -25,7 +25,7 @@
 #include "otutil.h"
 
 gboolean
-ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable, GError **error)
+ostree_builtin_trivial_httpd (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 {
   g_autoptr(GPtrArray) new_argv = g_ptr_array_new ();
 

src/ostree/ot-builtins.h

@@ -27,7 +27,7 @@
 
 G_BEGIN_DECLS
 
-#define BUILTINPROTO(name) gboolean ostree_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error)
+#define BUILTINPROTO(name) gboolean ostree_builtin_ ## name (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error)
 
 BUILTINPROTO(admin);
 BUILTINPROTO(cat);

src/ostree/ot-main.c

@@ -64,7 +64,11 @@ ostree_option_context_new_with_commands (OstreeCommand *commands)
 
   while (commands->name != NULL)
     {
-      g_string_append_printf (summary, "\n  %s", commands->name);
+      g_string_append_printf (summary, "\n  %-18s", commands->name);
+
+      if (commands->description != NULL )
+        g_string_append_printf (summary, "%s", commands->description);
+
       commands++;
     }
 
@@ -163,13 +167,11 @@ ostree_run (int    argc,
 
   if (!command->fn)
     {
-      g_autoptr(GOptionContext) context = NULL;
-      g_autofree char *help = NULL;
-
-      context = ostree_option_context_new_with_commands (commands);
+      g_autoptr(GOptionContext) context =
+        ostree_option_context_new_with_commands (commands);
 
       /* This will not return for some options (e.g. --version). */
-      if (ostree_option_context_parse (context, NULL, &argc, &argv, OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, &error))
+      if (ostree_option_context_parse (context, NULL, &argc, &argv, NULL, NULL, cancellable, &error))
         {
           if (command_name == NULL)
             {
@@ -191,8 +193,8 @@ ostree_run (int    argc,
   prgname = g_strdup_printf ("%s %s", g_get_prgname (), command_name);
   g_set_prgname (prgname);
 #endif
-
-  if (!command->fn (argc, argv, cancellable, &error))
+  OstreeCommandInvocation invocation = { .command = command };
+  if (!command->fn (argc, argv, &invocation, cancellable, &error))
     goto out;
 
   success = TRUE;
@@ -294,13 +296,39 @@ ostree_option_context_parse (GOptionContext *context,
                              const GOptionEntry *main_entries,
                              int *argc,
                              char ***argv,
-                             OstreeBuiltinFlags flags,
+                             OstreeCommandInvocation *invocation,
                              OstreeRepo **out_repo,
                              GCancellable *cancellable,
                              GError **error)
 {
   g_autoptr(OstreeRepo) repo = NULL;
+  /* When invocation is NULL,  do not fetch repo */
+  const OstreeBuiltinFlags flags = invocation ? invocation->command->flags : OSTREE_BUILTIN_FLAG_NO_REPO;
 
+  if (invocation && invocation->command->description != NULL)
+    {
+      const char *context_summary = g_option_context_get_summary (context);
+
+      /* If the summary is originally empty, we set the description, but
+       * for root commands(command with subcommands), we want to prepend
+       * the description to the existing summary string
+       */
+      if (context_summary == NULL)
+        g_option_context_set_summary (context, invocation->command->description);
+      else
+        {
+          /* TODO: remove this part once we deduplicate the ostree_option_context_new_with_commands
+           * function from other root commands( command with subcommands). Because
+           * we can directly add the summary inside the ostree_option_context_new_with_commands function.
+           */
+          g_autoptr(GString) new_summary_string = g_string_new (context_summary);
+
+          g_string_prepend (new_summary_string, "\n\n");
+          g_string_prepend (new_summary_string, invocation->command->description);
+
+          g_option_context_set_summary (context, new_summary_string->str);
+        }
+    }
   /* Entries are listed in --help output in the order added.  We add the
    * main entries ourselves so that we can add the --repo entry first. */
 
@@ -364,6 +392,7 @@ ostree_admin_option_context_parse (GOptionContext *context,
                                    int *argc,
                                    char ***argv,
                                    OstreeAdminBuiltinFlags flags,
+                                   OstreeCommandInvocation *invocation,
                                    OstreeSysroot **out_sysroot,
                                    GCancellable *cancellable,
                                    GError **error)
@@ -374,7 +403,7 @@ ostree_admin_option_context_parse (GOptionContext *context,
   g_option_context_add_main_entries (context, global_admin_entries, NULL);
 
   if (!ostree_option_context_parse (context, main_entries, argc, argv,
-                                    OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, error))
+                                    invocation, NULL, cancellable, error))
     return FALSE;
 
   if (!opt_print_current_dir && (flags & OSTREE_ADMIN_BUILTIN_FLAG_NO_SYSROOT))