From 722e143ee1109717064dc7ef06777f66a80116dd Mon Sep 17 00:00:00 2001
From: Mario Sanchez Prada <mario@endlessm.com>
Date: Thu, 15 Dec 2016 19:40:18 +0000
Subject: [PATCH 01/27] static-delta: Pretend that world unreadable objects are
 new objects

This will prevent including in the delta the bits to update files that
are not world readable, so that we don't run into a permissions problem
when applying the deltas from a bare-user repository that has a bare
repository set as its parent.

This is the case for Endless when updating flatpak runtimes, as the
temporary directory created in ~/.local/share/flatpak/system-cache will
be of type bare-user with its parent set to /var/lib/flatpak which is a
bare repository in EOS, as it's shared with the one at /ostree/repo.

https://phabricator.endlessm.com/T14159

Closes: #634
Approved by: cgwalters
---
 .../ostree-repo-static-delta-compilation.c    | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c
index 1611d19e..657e8676 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -843,6 +843,25 @@ process_one_bsdiff (OstreeRepo                       *repo,
   return ret;
 }
 
+static gboolean
+check_object_world_readable (OstreeRepo   *repo,
+                             const char   *checksum,
+                             gboolean     *out_readable,
+                             GCancellable *cancellable,
+                             GError      **error)
+{
+  g_autoptr(GFileInfo) finfo = NULL;
+  guint32 mode;
+
+  if (!ostree_repo_load_file (repo, checksum, NULL, &finfo, NULL,
+                              cancellable, error))
+    return FALSE;
+
+  mode = g_file_info_get_attribute_uint32 (finfo, "unix::mode");
+  *out_readable = (mode & S_IROTH);
+  return TRUE;
+}
+
 static gboolean 
 generate_delta_lowlatency (OstreeRepo                       *repo,
                            const char                       *from,
@@ -973,6 +992,22 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
       const char *from_checksum = value;
       ContentRollsum *rollsum;
       ContentBsdiff *bsdiff;
+      gboolean from_world_readable = FALSE;
+
+      /* We only want to include in the delta objects that we are sure will
+       * be readable by the client when applying the delta, regardless its
+       * access privileges, so that we don't run into permissions problems
+       * when the client is trying to update a bare-user repository with a
+       * bare repository defined as its parent.
+       */
+      if (!check_object_world_readable (repo, from_checksum, &from_world_readable, cancellable, error))
+        goto out;
+      if (!from_world_readable)
+        {
+          g_hash_table_iter_steal (&hashiter);
+          g_hash_table_add (new_reachable_regfile_content, (char*)from_checksum);
+          continue;
+        }
 
       if (!try_content_rollsum (repo, opts, from_checksum, to_checksum,
                                 &rollsum, cancellable, error))

From 169a629345ccf42d211005c38e98a246c3f87e21 Mon Sep 17 00:00:00 2001
From: Dan Nicholson <nicholson@endlessm.com>
Date: Wed, 21 Dec 2016 05:58:19 -0600
Subject: [PATCH 02/27] repo: Fix indentation

Closes: #635
Approved by: cgwalters
---
 src/libostree/ostree-repo.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 1c4ae663..92de1ed5 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -2456,12 +2456,12 @@ list_loose_objects_at (OstreeRepo             *self,
             continue;
         }
 
-        key = ostree_object_name_serialize (buf, objtype);
-        value = g_variant_new ("(b@as)",
-                               TRUE, g_variant_new_strv (NULL, 0));
-        /* transfer ownership */
-        g_hash_table_replace (inout_objects, key,
-                              g_variant_ref_sink (value));
+      key = ostree_object_name_serialize (buf, objtype);
+      value = g_variant_new ("(b@as)",
+                             TRUE, g_variant_new_strv (NULL, 0));
+      /* transfer ownership */
+      g_hash_table_replace (inout_objects, key,
+                            g_variant_ref_sink (value));
     }
 
   ret = TRUE;

From dd3cda401beed444ff2229f1c56e74b2b9fa922e Mon Sep 17 00:00:00 2001
From: Dan Nicholson <nicholson@endlessm.com>
Date: Wed, 21 Dec 2016 05:59:08 -0600
Subject: [PATCH 03/27] repo: Fix object list keys ownership

ostree_object_name_serialize returns a floating ref, so sink it before
adding it to the hash table so it can properly be freed later when the
hash table is destroyed.

This is particularly a problem for pygobject, which sinks the refs on
variants as it marshals them to native python types. If the ref isn't
already sunk, then the ref count won't increase and a critical warning
will be raised when both the hash table and pygobject try to unref it.

Closes: #635
Approved by: cgwalters
---
 src/libostree/ostree-repo.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 92de1ed5..35136c76 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -2460,7 +2460,7 @@ list_loose_objects_at (OstreeRepo             *self,
       value = g_variant_new ("(b@as)",
                              TRUE, g_variant_new_strv (NULL, 0));
       /* transfer ownership */
-      g_hash_table_replace (inout_objects, key,
+      g_hash_table_replace (inout_objects, g_variant_ref_sink (key),
                             g_variant_ref_sink (value));
     }
 

From 300752e55aa42ef8b60c7a0765b5cd14e1e9b98e Mon Sep 17 00:00:00 2001
From: Dan Nicholson <nicholson@endlessm.com>
Date: Thu, 22 Sep 2016 11:53:19 -0700
Subject: [PATCH 04/27] repo: Fix list_objects annotations

Without the element-type annotations, bindings don't know how to handle
the elements of the hash table. Since the table is created with destroy
functions, the caller does not own the elements, so transfer container
is used.

Closes: #635
Approved by: cgwalters
---
 src/libostree/ostree-repo.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 35136c76..3b3aa664 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -3548,7 +3548,8 @@ ostree_repo_load_commit (OstreeRepo            *self,
  * ostree_repo_list_objects:
  * @self: Repo
  * @flags: Flags controlling enumeration
- * @out_objects: (out): Map of serialized object name to variant data
+ * @out_objects: (out) (transfer container) (element-type GVariant GVariant):
+ * Map of serialized object name to variant data
  * @cancellable: Cancellable
  * @error: Error
  *
@@ -3605,7 +3606,8 @@ ostree_repo_list_objects (OstreeRepo                  *self,
  * ostree_repo_list_commit_objects_starting_with:
  * @self: Repo
  * @start: List commits starting with this checksum
- * @out_commits: Array of GVariants
+ * @out_commits: (out) (transfer container) (element-type GVariant GVariant):
+ * Map of serialized commit name to variant data
  * @cancellable: Cancellable
  * @error: Error
  *

From 4d0b9be17553ed6bd59b89ab260fbd95d17588cf Mon Sep 17 00:00:00 2001
From: Paul van Tilburg <paul.van.tilburg@leftclick.eu>
Date: Wed, 21 Dec 2016 15:39:45 +0000
Subject: [PATCH 05/27] admin: Use execlp() to look for systemctl as the shell
 would

Closes: #637
Approved by: cgwalters
---
 src/ostree/ot-admin-functions.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ostree/ot-admin-functions.c b/src/ostree/ot-admin-functions.c
index ed4dfdfb..7ba2207e 100644
--- a/src/ostree/ot-admin-functions.c
+++ b/src/ostree/ot-admin-functions.c
@@ -164,7 +164,7 @@ ot_admin_execve_reboot (OstreeSysroot *sysroot, GError **error)
       
   if (g_file_equal (ostree_sysroot_get_path (sysroot), real_sysroot))
     {
-      if (execl ("systemctl", "systemctl", "reboot", NULL) < 0)
+      if (execlp ("systemctl", "systemctl", "reboot", NULL) < 0)
         {
           glnx_set_error_from_errno (error);
           return FALSE;

From eceb165013a2bba8f4aaca34aa8dc01f4633d152 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Fri, 23 Dec 2016 09:59:30 -0500
Subject: [PATCH 06/27] docs: Fix ostree.version -> version

I think originally I had envisioned this as `ostree.version`, but at the last
minute we changed it to just `version`. That's what all of the code uses, so
let's fix the docs.

Closes: #638
Approved by: paulvt
---
 docs/manual/repository-management.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md
index 5688a582..64b2f6e1 100644
--- a/docs/manual/repository-management.md
+++ b/docs/manual/repository-management.md
@@ -154,7 +154,7 @@ and generate a new commit in our prod repository:
 checksum=$(ostree --repo=repo-dev rev-parse exampleos/x86_64/stage-3-pass/standard`)
 ostree --repo=repo-prod pull-local repo-dev ${checksum}
 ostree --repo=repo-prod commit -b exampleos/x86_64/standard \
-       -s 'Release 1.2.3' --add-metadata-string=ostree.version=1.2.3 \
+       -s 'Release 1.2.3' --add-metadata-string=version=1.2.3 \
 	   --tree=ref=${checksum}
 ```
 
@@ -171,7 +171,7 @@ links to avoid copying any content at all - making the process very
 fast.
 
 Another interesting thing to notice here is that we're adding an
-`ostree.version` metadata string to the commit.  This is an optional
+`version` metadata string to the commit.  This is an optional
 piece of metadata, but we are encouraging its use in the OSTree
 ecosystem of tools.  Commands like `ostree admin status` show it by
 default.

From af560047a2460bf7e75a5e343b9bb338a52131e2 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 22 Dec 2016 17:57:04 -0500
Subject: [PATCH 07/27] fetcher: Hoist core "mirrored request" API to public

This is in preparation for the libcurl port. We're basically making public what
we had internally. The next step here is to create `ostree-fetcher-util.[ch]`
that only operates in terms of this lower level API.

Also drop the `_mirrored` from the function name since it's
the default now.

Closes: #636
Approved by: jlebon
---
 src/libostree/ostree-fetcher.c   | 117 ++++++++++++++-----------------
 src/libostree/ostree-fetcher.h   |  29 +++++---
 src/libostree/ostree-repo-pull.c |  37 +++++-----
 3 files changed, 87 insertions(+), 96 deletions(-)

diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 63942166..c69bbefa 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1179,17 +1179,16 @@ on_request_sent (GObject        *object,
   g_object_unref (task);
 }
 
-static void
-ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
-                                          GPtrArray             *mirrorlist,
-                                          const char            *filename,
-                                          gboolean               is_stream,
-                                          guint64                max_size,
-                                          int                    priority,
-                                          GCancellable          *cancellable,
-                                          GAsyncReadyCallback    callback,
-                                          gpointer               user_data,
-                                          gpointer               source_tag)
+void
+_ostree_fetcher_request_async (OstreeFetcher         *self,
+                               GPtrArray             *mirrorlist,
+                               const char            *filename,
+                               OstreeFetcherRequestFlags flags,
+                               guint64                max_size,
+                               int                    priority,
+                               GCancellable          *cancellable,
+                               GAsyncReadyCallback    callback,
+                               gpointer               user_data)
 {
   g_autoptr(GTask) task = NULL;
   OstreeFetcherPendingURI *pending;
@@ -1205,10 +1204,10 @@ ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
   pending->mirrorlist = g_ptr_array_ref (mirrorlist);
   pending->filename = g_strdup (filename);
   pending->max_size = max_size;
-  pending->is_stream = is_stream;
+  pending->is_stream = (flags & OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL) == 0;
 
   task = g_task_new (self, cancellable, callback, user_data);
-  g_task_set_source_tag (task, source_tag);
+  g_task_set_source_tag (task, _ostree_fetcher_request_async);
   g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref);
 
   /* We'll use the GTask priority for our own priority queue. */
@@ -1220,60 +1219,45 @@ ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
                            (GDestroyNotify) g_object_unref);
 }
 
-void
-_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher         *self,
-                                                     GPtrArray             *mirrorlist,
-                                                     const char            *filename,
-                                                     guint64                max_size,
-                                                     int                    priority,
-                                                     GCancellable          *cancellable,
-                                                     GAsyncReadyCallback    callback,
-                                                     gpointer               user_data)
+gboolean
+_ostree_fetcher_request_finish (OstreeFetcher         *self,
+                                GAsyncResult          *result,
+                                char                 **out_filename,
+                                GInputStream         **out_stream,
+                                GError               **error)
 {
-  ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE,
-                                            max_size, priority, cancellable,
-                                            callback, user_data,
-                                            _ostree_fetcher_mirrored_request_with_partial_async);
-}
+  GTask *task;
+  OstreeFetcherPendingURI *pending;
+  gpointer ret;
 
-char *
-_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher         *self,
-                                                      GAsyncResult          *result,
-                                                      GError               **error)
-{
-  g_return_val_if_fail (g_task_is_valid (result, self), NULL);
-  g_return_val_if_fail (g_async_result_is_tagged (result,
-                        _ostree_fetcher_mirrored_request_with_partial_async), NULL);
+  g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+  g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
 
-  return g_task_propagate_pointer (G_TASK (result), error);
-}
+  /* Special dance to implement
+    enum FetchResult {
+       Filename(String path),
+       Membuf(uint8[])
+    } in Rust terms
+  */
+  task = (GTask*)result;
+  pending = g_task_get_task_data (task);
 
-static void
-ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher         *self,
-                                          GPtrArray             *mirrorlist,
-                                          const char            *filename,
-                                          guint64                max_size,
-                                          int                    priority,
-                                          GCancellable          *cancellable,
-                                          GAsyncReadyCallback    callback,
-                                          gpointer               user_data)
-{
-  ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE,
-                                            max_size, priority, cancellable,
-                                            callback, user_data,
-                                            ostree_fetcher_stream_mirrored_uri_async);
-}
+  ret = g_task_propagate_pointer (task, error);
+  if (!ret)
+    return FALSE;
 
-static GInputStream *
-ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher         *self,
-                                           GAsyncResult          *result,
-                                           GError               **error)
-{
-  g_return_val_if_fail (g_task_is_valid (result, self), NULL);
-  g_return_val_if_fail (g_async_result_is_tagged (result,
-                        ostree_fetcher_stream_mirrored_uri_async), NULL);
+  if (pending->is_stream)
+    {
+      g_assert (out_stream);
+      *out_stream = ret;
+    }
+  else
+    {
+      g_assert (out_filename);
+      *out_filename = ret;
+    }
 
-  return g_task_propagate_pointer (G_TASK (result), error);
+  return TRUE;
 }
 
 guint64
@@ -1322,8 +1306,9 @@ fetch_uri_sync_on_complete (GObject        *object,
 {
   FetchUriSyncData *data = user_data;
 
-  data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object,
-                                                                    result, data->error);
+  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
+                                        result, NULL, &data->result_stream,
+                                        data->error);
   data->done = TRUE;
 }
 
@@ -1356,9 +1341,9 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
   data.done = FALSE;
   data.error = error;
 
-  ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size,
-                                   OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                   fetch_uri_sync_on_complete, &data);
+  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
+                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
+                                 fetch_uri_sync_on_complete, &data);
   while (!data.done)
     g_main_context_iteration (mainctx, TRUE);
 
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 8e282e24..124a02bf 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -103,18 +103,25 @@ void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
 
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
-void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher         *self,
-                                                          GPtrArray             *mirrorlist,
-                                                          const char            *filename,
-                                                          guint64                max_size,
-                                                          int                    priority,
-                                                          GCancellable          *cancellable,
-                                                          GAsyncReadyCallback    callback,
-                                                          gpointer               user_data);
+typedef enum {
+  OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL = (1 << 0)
+} OstreeFetcherRequestFlags;
 
-char *_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
-                                                            GAsyncResult  *result,
-                                                            GError       **error);
+void _ostree_fetcher_request_async (OstreeFetcher         *self,
+                                    GPtrArray             *mirrorlist,
+                                    const char            *filename,
+                                    OstreeFetcherRequestFlags flags,
+                                    guint64                max_size,
+                                    int                    priority,
+                                    GCancellable          *cancellable,
+                                    GAsyncReadyCallback    callback,
+                                    gpointer               user_data);
+
+gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
+                                         GAsyncResult  *result,
+                                         char         **out_filename,
+                                         GInputStream **out_stream,
+                                         GError       **error);
 
 gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
                                                      GPtrArray     *mirrorlist,
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 9c99dc4f..32ad5fc0 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -707,8 +707,7 @@ content_fetch_on_complete (GObject        *object,
   OstreeObjectType objtype;
   gboolean free_fetch_data = TRUE;
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     goto out;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -841,8 +840,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", checksum_obj,
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
@@ -982,8 +980,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     goto out;
 
   fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
@@ -1382,12 +1379,13 @@ enqueue_one_object_request (OtPullData        *pull_data,
   else
     expected_max_size = 0;
 
-  _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist,
-                                                       obj_subpath, expected_max_size,
-                                                       is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
-                                                               : OSTREE_REPO_PULL_CONTENT_PRIORITY,
-                                                       pull_data->cancellable,
-                                                       is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
+  _ostree_fetcher_request_async (pull_data->fetcher, mirrorlist,
+                                 obj_subpath, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
+                                 expected_max_size,
+                                 is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
+                                 : OSTREE_REPO_PULL_CONTENT_PRIORITY,
+                                 pull_data->cancellable,
+                                 is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
 }
 
 static gboolean
@@ -1735,13 +1733,14 @@ process_one_static_delta (OtPullData   *pull_data,
         }
       else
         {
-          _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher,
-                                                               pull_data->content_mirrorlist,
-                                                               deltapart_path, size,
-                                                               OSTREE_FETCHER_DEFAULT_PRIORITY,
-                                                               pull_data->cancellable,
-                                                               static_deltapart_fetch_on_complete,
-                                                               fetch_data);
+          _ostree_fetcher_request_async (pull_data->fetcher,
+                                         pull_data->content_mirrorlist,
+                                         deltapart_path, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
+                                         size,
+                                         OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                         pull_data->cancellable,
+                                         static_deltapart_fetch_on_complete,
+                                         fetch_data);
           pull_data->n_outstanding_deltapart_fetches++;
         }
     }

From 9d0d0a26dbfa55080e580ed0bc7bb6c50ca46c21 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 22 Dec 2016 20:34:07 -0500
Subject: [PATCH 08/27] fetcher: Move high level functions into "fetcher-util"

Conceptually these now lay on top of the core API, and don't reference libsoup.
This is preparation for libcurl porting, but it's also just generally better.

Closes: #636
Approved by: jlebon
---
 Makefile-libostree.am               |   2 +
 src/libostree/ostree-fetcher-util.c | 140 ++++++++++++++++++++++++++++
 src/libostree/ostree-fetcher-util.h |  49 ++++++++++
 src/libostree/ostree-fetcher.c      | 113 ----------------------
 src/libostree/ostree-fetcher.h      |  18 ----
 src/libostree/ostree-metalink.c     |   1 +
 src/libostree/ostree-repo-pull.c    |   1 +
 7 files changed, 193 insertions(+), 131 deletions(-)
 create mode 100644 src/libostree/ostree-fetcher-util.c
 create mode 100644 src/libostree/ostree-fetcher-util.h

diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index a7d7b11b..88210c75 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -161,6 +161,8 @@ if USE_LIBSOUP
 libostree_1_la_SOURCES += \
 	src/libostree/ostree-fetcher.h \
 	src/libostree/ostree-fetcher.c \
+	src/libostree/ostree-fetcher-util.h \
+	src/libostree/ostree-fetcher-util.c \
 	src/libostree/ostree-metalink.h \
 	src/libostree/ostree-metalink.c \
 	$(NULL)
diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
new file mode 100644
index 00000000..bc97674d
--- /dev/null
+++ b/src/libostree/ostree-fetcher-util.c
@@ -0,0 +1,140 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2017 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <gio/gfiledescriptorbased.h>
+#include <gio/gunixoutputstream.h>
+
+#include "ostree-fetcher-util.h"
+#include "otutil.h"
+
+typedef struct
+{
+  GInputStream   *result_stream;
+  gboolean         done;
+  GError         **error;
+}
+  FetchUriSyncData;
+
+static void
+fetch_uri_sync_on_complete (GObject        *object,
+                            GAsyncResult   *result,
+                            gpointer        user_data)
+{
+  FetchUriSyncData *data = user_data;
+
+  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
+                                        result, NULL, &data->result_stream,
+                                        data->error);
+  data->done = TRUE;
+}
+
+gboolean
+_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
+                                            GPtrArray     *mirrorlist,
+                                            const char     *filename,
+                                            gboolean        add_nul,
+                                            gboolean        allow_noent,
+                                            GBytes         **out_contents,
+                                            guint64        max_size,
+                                            GCancellable   *cancellable,
+                                            GError         **error)
+{
+  gboolean ret = FALSE;
+  const guint8 nulchar = 0;
+  g_autoptr(GMemoryOutputStream) buf = NULL;
+  g_autoptr(GMainContext) mainctx = NULL;
+  FetchUriSyncData data;
+  g_assert (error != NULL);
+
+  data.result_stream = NULL;
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  mainctx = g_main_context_new ();
+  g_main_context_push_thread_default (mainctx);
+
+  data.done = FALSE;
+  data.error = error;
+
+  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
+                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
+                                 fetch_uri_sync_on_complete, &data);
+  while (!data.done)
+    g_main_context_iteration (mainctx, TRUE);
+
+  if (!data.result_stream)
+    {
+      if (allow_noent)
+        {
+          if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+            {
+              g_clear_error (error);
+              ret = TRUE;
+              *out_contents = NULL;
+            }
+        }
+      goto out;
+    }
+
+  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
+                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+                              cancellable, error) < 0)
+    goto out;
+
+  if (add_nul)
+    {
+      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
+        goto out;
+    }
+
+  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
+ out:
+  if (mainctx)
+    g_main_context_pop_thread_default (mainctx);
+  g_clear_object (&(data.result_stream));
+  return ret;
+}
+
+/* Helper for callers who just want to fetch single one-off URIs */
+gboolean
+_ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
+                                       OstreeFetcherURI *uri,
+                                       gboolean        add_nul,
+                                       gboolean        allow_noent,
+                                       GBytes         **out_contents,
+                                       guint64        max_size,
+                                       GCancellable   *cancellable,
+                                       GError         **error)
+{
+  g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
+  g_ptr_array_add (mirrorlist, uri); /* no transfer */
+  return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
+                                                     add_nul, allow_noent,
+                                                     out_contents, max_size,
+                                                     cancellable, error);
+}
diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h
new file mode 100644
index 00000000..0f25dc30
--- /dev/null
+++ b/src/libostree/ostree-fetcher-util.h
@@ -0,0 +1,49 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2016 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include "ostree-fetcher.h"
+
+G_BEGIN_DECLS
+
+gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
+                                                     GPtrArray     *mirrorlist,
+                                                     const char    *filename,
+                                                     gboolean       add_nul,
+                                                     gboolean       allow_noent,
+                                                     GBytes         **out_contents,
+                                                     guint64        max_size,
+                                                     GCancellable   *cancellable,
+                                                     GError         **error);
+
+gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
+                                                OstreeFetcherURI *uri,
+                                                gboolean       add_nul,
+                                                gboolean       allow_noent,
+                                                GBytes         **out_contents,
+                                                guint64        max_size,
+                                                GCancellable   *cancellable,
+                                                GError         **error);
+G_END_DECLS
+
+#endif
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index c69bbefa..7917f729 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1291,119 +1291,6 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher       *self)
   return ret;
 }
 
-typedef struct
-{
-  GInputStream   *result_stream;
-  gboolean         done;
-  GError         **error;
-}
-FetchUriSyncData;
-
-static void
-fetch_uri_sync_on_complete (GObject        *object,
-                            GAsyncResult   *result,
-                            gpointer        user_data)
-{
-  FetchUriSyncData *data = user_data;
-
-  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
-                                        result, NULL, &data->result_stream,
-                                        data->error);
-  data->done = TRUE;
-}
-
-gboolean
-_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
-                                            GPtrArray     *mirrorlist,
-                                            const char     *filename,
-                                            gboolean        add_nul,
-                                            gboolean        allow_noent,
-                                            GBytes         **out_contents,
-                                            guint64        max_size,
-                                            GCancellable   *cancellable,
-                                            GError         **error)
-{
-  gboolean ret = FALSE;
-  const guint8 nulchar = 0;
-  g_autoptr(GMemoryOutputStream) buf = NULL;
-  g_autoptr(GMainContext) mainctx = NULL;
-  FetchUriSyncData data;
-  g_assert (error != NULL);
-
-  data.result_stream = NULL;
-
-  if (g_cancellable_set_error_if_cancelled (cancellable, error))
-    return FALSE;
-
-  mainctx = g_main_context_new ();
-  g_main_context_push_thread_default (mainctx);
-
-  data.done = FALSE;
-  data.error = error;
-
-  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
-                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                 fetch_uri_sync_on_complete, &data);
-  while (!data.done)
-    g_main_context_iteration (mainctx, TRUE);
-
-  if (!data.result_stream)
-    {
-      if (allow_noent)
-        {
-          if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (error);
-              ret = TRUE;
-              *out_contents = NULL;
-            }
-        }
-      goto out;
-    }
-
-  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
-                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                              cancellable, error) < 0)
-    goto out;
-
-  if (add_nul)
-    {
-      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
-        goto out;
-    }
-
-  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
-    goto out;
-
-  ret = TRUE;
-  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
- out:
-  if (mainctx)
-    g_main_context_pop_thread_default (mainctx);
-  g_clear_object (&(data.result_stream));
-  return ret;
-}
-
-/* Helper for callers who just want to fetch single one-off URIs */
-gboolean
-_ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
-                                       OstreeFetcherURI *uri,
-                                       gboolean        add_nul,
-                                       gboolean        allow_noent,
-                                       GBytes         **out_contents,
-                                       guint64        max_size,
-                                       GCancellable   *cancellable,
-                                       GError         **error)
-{
-  g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
-  g_ptr_array_add (mirrorlist, uri); /* no transfer */
-  return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
-                                                     add_nul, allow_noent,
-                                                     out_contents, max_size,
-                                                     cancellable, error);
-}
-
 void
 _ostree_fetcher_uri_free (OstreeFetcherURI *uri)
 {
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 124a02bf..10ec51dc 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -123,24 +123,6 @@ gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
                                          GInputStream **out_stream,
                                          GError       **error);
 
-gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
-                                                     GPtrArray     *mirrorlist,
-                                                     const char    *filename,
-                                                     gboolean       add_nul,
-                                                     gboolean       allow_noent,
-                                                     GBytes         **out_contents,
-                                                     guint64        max_size,
-                                                     GCancellable   *cancellable,
-                                                     GError         **error);
-
-gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
-                                                OstreeFetcherURI *uri,
-                                                gboolean       add_nul,
-                                                gboolean       allow_noent,
-                                                GBytes         **out_contents,
-                                                guint64        max_size,
-                                                GCancellable   *cancellable,
-                                                GError         **error);
 G_END_DECLS
 
 #endif
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
index ee52e51b..6afae59e 100644
--- a/src/libostree/ostree-metalink.c
+++ b/src/libostree/ostree-metalink.c
@@ -21,6 +21,7 @@
 #include "config.h"
 
 #include "ostree-metalink.h"
+#include "ostree-fetcher-util.h"
 #include <gio/gfiledescriptorbased.h>
 
 #include "otutil.h"
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 32ad5fc0..100d9304 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -32,6 +32,7 @@
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
+#include "ostree-fetcher-util.h"
 #include "ot-fs-utils.h"
 
 #include <gio/gunixinputstream.h>

From 21aca3fa83d7365376f6be2056fc1bac33f9998d Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 28 Dec 2016 14:43:28 -0500
Subject: [PATCH 09/27] fetcher: Split lowlevel API into file/membuf variants

The previous commit introduced a single low level API - however,
we can do things in a more optimal way for the curl backend if
we drop the "streaming API" variant.  Currently, we only use
it to synchronously splice into a memory buffer...which is pretty
silly when we could just do that in the backend.

The only tweak here is that we have an "add NUL character" flag that is
(possibly) needed when fetching into a membuf.

The code here ends up being better I think, since we avoid the double return
value for the `_finish()` invocation, and now most of the fetcher code (in the
soup case) writes to a `GOutputStream` consistently.

This will again make things easier for a curl backend.

Closes: #636
Approved by: jlebon
---
 src/libostree/ostree-fetcher-util.c |  40 ++----
 src/libostree/ostree-fetcher.c      | 210 ++++++++++++++++++----------
 src/libostree/ostree-fetcher.h      |  44 ++++--
 src/libostree/ostree-repo-pull.c    |  34 +++--
 4 files changed, 194 insertions(+), 134 deletions(-)

diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
index bc97674d..b8af972a 100644
--- a/src/libostree/ostree-fetcher-util.c
+++ b/src/libostree/ostree-fetcher-util.c
@@ -28,7 +28,7 @@
 
 typedef struct
 {
-  GInputStream   *result_stream;
+  GBytes          *result_buf;
   gboolean         done;
   GError         **error;
 }
@@ -41,9 +41,9 @@ fetch_uri_sync_on_complete (GObject        *object,
 {
   FetchUriSyncData *data = user_data;
 
-  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
-                                        result, NULL, &data->result_stream,
-                                        data->error);
+  (void)_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)object,
+                                                  result, &data->result_buf,
+                                                  data->error);
   data->done = TRUE;
 }
 
@@ -59,13 +59,11 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
                                             GError         **error)
 {
   gboolean ret = FALSE;
-  const guint8 nulchar = 0;
-  g_autoptr(GMemoryOutputStream) buf = NULL;
   g_autoptr(GMainContext) mainctx = NULL;
   FetchUriSyncData data;
   g_assert (error != NULL);
 
-  data.result_stream = NULL;
+  memset (&data, 0, sizeof (data));
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
@@ -76,13 +74,14 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
   data.done = FALSE;
   data.error = error;
 
-  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
-                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                 fetch_uri_sync_on_complete, &data);
+  _ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename,
+                                     add_nul ? OSTREE_FETCHER_REQUEST_NUL_TERMINATION : 0,
+                                     max_size, OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                     cancellable, fetch_uri_sync_on_complete, &data);
   while (!data.done)
     g_main_context_iteration (mainctx, TRUE);
 
-  if (!data.result_stream)
+  if (!data.result_buf)
     {
       if (allow_noent)
         {
@@ -96,27 +95,12 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
       goto out;
     }
 
-  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
-                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                              cancellable, error) < 0)
-    goto out;
-
-  if (add_nul)
-    {
-      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
-        goto out;
-    }
-
-  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
-    goto out;
-
   ret = TRUE;
-  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
+  *out_contents = g_steal_pointer (&data.result_buf);
  out:
   if (mainctx)
     g_main_context_pop_thread_default (mainctx);
-  g_clear_object (&(data.result_stream));
+  g_clear_pointer (&data.result_buf, (GDestroyNotify)g_bytes_unref);
   return ret;
 }
 
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 7917f729..cee33186 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -22,6 +22,7 @@
 
 #include "config.h"
 
+#include <gio/gio.h>
 #include <gio/gfiledescriptorbased.h>
 #include <gio/gunixoutputstream.h>
 #define LIBSOUP_USE_UNSTABLE_REQUEST_API
@@ -90,7 +91,8 @@ typedef struct {
 
   SoupRequest *request;
 
-  gboolean is_stream;
+  gboolean is_membuf;
+  OstreeFetcherRequestFlags flags;
   GInputStream *request_body;
   char *out_tmpfile;
   GOutputStream *out_stream;
@@ -468,7 +470,7 @@ session_thread_request_uri (ThreadClosure *thread_closure,
         soup_message_headers_append (msg->request_headers, key, value);
     }
 
-  if (pending->is_stream)
+  if (pending->is_membuf)
     {
       soup_request_send_async (pending->request,
                                cancellable,
@@ -855,6 +857,16 @@ finish_stream (OstreeFetcherPendingURI *pending,
    */
   if (pending->out_stream)
     {
+      if ((pending->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0)
+        {
+          const guint8 nulchar = 0;
+          gsize bytes_written;
+
+          if (!g_output_stream_write_all (pending->out_stream, &nulchar, 1, &bytes_written,
+                                          cancellable, error))
+            goto out;
+        }
+
       if (!g_output_stream_close (pending->out_stream, cancellable, error))
         goto out;
 
@@ -864,30 +876,37 @@ finish_stream (OstreeFetcherPendingURI *pending,
       g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
     }
 
-  pending->state = OSTREE_FETCHER_STATE_COMPLETE;
-  if (fstatat (pending->thread_closure->tmpdir_dfd,
-               pending->out_tmpfile,
-               &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
+  if (!pending->is_membuf)
     {
-      glnx_set_error_from_errno (error);
-      goto out;
+      if (fstatat (pending->thread_closure->tmpdir_dfd,
+                   pending->out_tmpfile,
+                   &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
+        {
+          glnx_set_error_from_errno (error);
+          goto out;
+        }
     }
 
+  pending->state = OSTREE_FETCHER_STATE_COMPLETE;
+
   /* Now that we've finished downloading, continue with other queued
    * requests.
    */
   session_thread_process_pending_queue (pending->thread_closure);
 
-  if (stbuf.st_size < pending->content_length)
+  if (!pending->is_membuf)
     {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
-      goto out;
-    }
-  else
-    {
-      g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
-      pending->thread_closure->total_downloaded += stbuf.st_size;
-      g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+      if (stbuf.st_size < pending->content_length)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
+          goto out;
+        }
+      else
+        {
+          g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
+          pending->thread_closure->total_downloaded += stbuf.st_size;
+          g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+        }
     }
 
   ret = TRUE;
@@ -973,9 +992,18 @@ on_stream_read (GObject        *object,
     {
       if (!finish_stream (pending, cancellable, &local_error))
         goto out;
-      g_task_return_pointer (task,
-                             g_strdup (pending->out_tmpfile),
-                             (GDestroyNotify) g_free);
+      if (pending->is_membuf)
+        {
+          g_task_return_pointer (task,
+                                 g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream*)pending->out_stream),
+                                 (GDestroyNotify) g_bytes_unref);
+        }
+      else
+        {
+          g_task_return_pointer (task,
+                                 g_strdup (pending->out_tmpfile),
+                                 (GDestroyNotify) g_free);
+        }
       remove_pending_rerun_queue (pending);
     }
   else
@@ -1045,23 +1073,15 @@ on_request_sent (GObject        *object,
   if (SOUP_IS_REQUEST_HTTP (object))
     {
       msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
-      if (msg->status_code == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE)
+      if (!pending->is_membuf &&
+          msg->status_code == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE)
         {
           // We already have the whole file, so just use it.
           pending->state = OSTREE_FETCHER_STATE_COMPLETE;
           (void) g_input_stream_close (pending->request_body, NULL, NULL);
-          if (pending->is_stream)
-            {
-              g_task_return_pointer (task,
-                                     g_object_ref (pending->request_body),
-                                     (GDestroyNotify) g_object_unref);
-            }
-          else
-            {
-              g_task_return_pointer (task,
-                                     g_strdup (pending->out_tmpfile),
-                                     (GDestroyNotify) g_free);
-            }
+          g_task_return_pointer (task,
+                                 g_strdup (pending->out_tmpfile),
+                                 (GDestroyNotify) g_free);
           remove_pending_rerun_queue (pending);
           goto out;
         }
@@ -1126,7 +1146,7 @@ on_request_sent (GObject        *object,
   
   pending->content_length = soup_request_get_content_length (pending->request);
 
-  if (!pending->is_stream)
+  if (!pending->is_membuf)
     {
       int oflags = O_CREAT | O_WRONLY | O_CLOEXEC;
       int fd;
@@ -1147,26 +1167,23 @@ on_request_sent (GObject        *object,
           goto out;
         }
       pending->out_stream = g_unix_output_stream_new (fd, TRUE);
-
-      g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
-      g_hash_table_add (pending->thread_closure->output_stream_set,
-                        g_object_ref (pending->out_stream));
-      g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
-
-      g_input_stream_read_bytes_async (pending->request_body,
-                                       8192, G_PRIORITY_DEFAULT,
-                                       cancellable,
-                                       on_stream_read,
-                                       g_object_ref (task));
     }
   else
     {
-      g_task_return_pointer (task,
-                             g_object_ref (pending->request_body),
-                             (GDestroyNotify) g_object_unref);
-      remove_pending_rerun_queue (pending);
+      pending->out_stream = g_memory_output_stream_new_resizable ();
     }
-  
+
+  g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
+  g_hash_table_add (pending->thread_closure->output_stream_set,
+                    g_object_ref (pending->out_stream));
+  g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+
+  g_input_stream_read_bytes_async (pending->request_body,
+                                   8192, G_PRIORITY_DEFAULT,
+                                   cancellable,
+                                   on_stream_read,
+                                   g_object_ref (task));
+
  out:
   if (local_error)
     {
@@ -1179,11 +1196,12 @@ on_request_sent (GObject        *object,
   g_object_unref (task);
 }
 
-void
+static void
 _ostree_fetcher_request_async (OstreeFetcher         *self,
                                GPtrArray             *mirrorlist,
                                const char            *filename,
                                OstreeFetcherRequestFlags flags,
+                               gboolean               is_membuf,
                                guint64                max_size,
                                int                    priority,
                                GCancellable          *cancellable,
@@ -1203,8 +1221,9 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
   pending->thread_closure = thread_closure_ref (self->thread_closure);
   pending->mirrorlist = g_ptr_array_ref (mirrorlist);
   pending->filename = g_strdup (filename);
+  pending->flags = flags;
   pending->max_size = max_size;
-  pending->is_stream = (flags & OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL) == 0;
+  pending->is_membuf = is_membuf;
 
   task = g_task_new (self, cancellable, callback, user_data);
   g_task_set_source_tag (task, _ostree_fetcher_request_async);
@@ -1219,12 +1238,26 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
                            (GDestroyNotify) g_object_unref);
 }
 
+void
+_ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
+                                    GPtrArray             *mirrorlist,
+                                    const char            *filename,
+                                    guint64                max_size,
+                                    int                    priority,
+                                    GCancellable          *cancellable,
+                                    GAsyncReadyCallback    callback,
+                                    gpointer               user_data)
+{
+  _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE,
+                                 max_size, priority, cancellable,
+                                 callback, user_data);
+}
+
 gboolean
-_ostree_fetcher_request_finish (OstreeFetcher         *self,
-                                GAsyncResult          *result,
-                                char                 **out_filename,
-                                GInputStream         **out_stream,
-                                GError               **error)
+_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
+                                           GAsyncResult  *result,
+                                           char         **out_filename,
+                                           GError       **error)
 {
   GTask *task;
   OstreeFetcherPendingURI *pending;
@@ -1233,12 +1266,6 @@ _ostree_fetcher_request_finish (OstreeFetcher         *self,
   g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
   g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
 
-  /* Special dance to implement
-    enum FetchResult {
-       Filename(String path),
-       Membuf(uint8[])
-    } in Rust terms
-  */
   task = (GTask*)result;
   pending = g_task_get_task_data (task);
 
@@ -1246,20 +1273,57 @@ _ostree_fetcher_request_finish (OstreeFetcher         *self,
   if (!ret)
     return FALSE;
 
-  if (pending->is_stream)
-    {
-      g_assert (out_stream);
-      *out_stream = ret;
-    }
-  else
-    {
-      g_assert (out_filename);
-      *out_filename = ret;
-    }
+  g_assert (!pending->is_membuf);
+  g_assert (out_filename);
+  *out_filename = ret;
 
   return TRUE;
 }
 
+void
+_ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
+                                   GPtrArray             *mirrorlist,
+                                   const char            *filename,
+                                   OstreeFetcherRequestFlags flags,
+                                   guint64                max_size,
+                                   int                    priority,
+                                   GCancellable          *cancellable,
+                                   GAsyncReadyCallback    callback,
+                                   gpointer               user_data)
+{
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
+                                 max_size, priority, cancellable,
+                                 callback, user_data);
+}
+
+gboolean
+_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
+                                          GAsyncResult  *result,
+                                          GBytes       **out_buf,
+                                          GError       **error)
+{
+  GTask *task;
+  OstreeFetcherPendingURI *pending;
+  gpointer ret;
+
+  g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+  g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
+
+  task = (GTask*)result;
+  pending = g_task_get_task_data (task);
+
+  ret = g_task_propagate_pointer (task, error);
+  if (!ret)
+    return FALSE;
+
+  g_assert (pending->is_membuf);
+  g_assert (out_buf);
+  *out_buf = ret;
+
+  return TRUE;
+}
+
+
 guint64
 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self)
 {
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 10ec51dc..a57907e4 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -103,25 +103,39 @@ void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
 
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
+void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
+                                         GPtrArray             *mirrorlist,
+                                         const char            *filename,
+                                         guint64                max_size,
+                                         int                    priority,
+                                         GCancellable          *cancellable,
+                                         GAsyncReadyCallback    callback,
+                                         gpointer               user_data);
+
+gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
+                                                    GAsyncResult  *result,
+                                                    char         **out_filename,
+                                                    GError       **error);
+
 typedef enum {
-  OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL = (1 << 0)
+  OSTREE_FETCHER_REQUEST_NUL_TERMINATION = (1 << 0)
 } OstreeFetcherRequestFlags;
 
-void _ostree_fetcher_request_async (OstreeFetcher         *self,
-                                    GPtrArray             *mirrorlist,
-                                    const char            *filename,
-                                    OstreeFetcherRequestFlags flags,
-                                    guint64                max_size,
-                                    int                    priority,
-                                    GCancellable          *cancellable,
-                                    GAsyncReadyCallback    callback,
-                                    gpointer               user_data);
+void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
+                                        GPtrArray             *mirrorlist,
+                                        const char            *filename,
+                                        OstreeFetcherRequestFlags flags,
+                                        guint64                max_size,
+                                        int                    priority,
+                                        GCancellable          *cancellable,
+                                        GAsyncReadyCallback    callback,
+                                        gpointer               user_data);
+
+gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
+                                                   GAsyncResult  *result,
+                                                   GBytes       **out_buf,
+                                                   GError       **error);
 
-gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
-                                         GAsyncResult  *result,
-                                         char         **out_filename,
-                                         GInputStream **out_stream,
-                                         GError       **error);
 
 G_END_DECLS
 
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 100d9304..92efd592 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -708,7 +708,7 @@ content_fetch_on_complete (GObject        *object,
   OstreeObjectType objtype;
   gboolean free_fetch_data = TRUE;
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     goto out;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -841,7 +841,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", checksum_obj,
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
@@ -981,7 +981,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     goto out;
 
   fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
@@ -1380,13 +1380,12 @@ enqueue_one_object_request (OtPullData        *pull_data,
   else
     expected_max_size = 0;
 
-  _ostree_fetcher_request_async (pull_data->fetcher, mirrorlist,
-                                 obj_subpath, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
-                                 expected_max_size,
-                                 is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
-                                 : OSTREE_REPO_PULL_CONTENT_PRIORITY,
-                                 pull_data->cancellable,
-                                 is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
+  _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
+                                      obj_subpath, expected_max_size,
+                                      is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
+                                      : OSTREE_REPO_PULL_CONTENT_PRIORITY,
+                                      pull_data->cancellable,
+                                      is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
 }
 
 static gboolean
@@ -1734,14 +1733,13 @@ process_one_static_delta (OtPullData   *pull_data,
         }
       else
         {
-          _ostree_fetcher_request_async (pull_data->fetcher,
-                                         pull_data->content_mirrorlist,
-                                         deltapart_path, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
-                                         size,
-                                         OSTREE_FETCHER_DEFAULT_PRIORITY,
-                                         pull_data->cancellable,
-                                         static_deltapart_fetch_on_complete,
-                                         fetch_data);
+          _ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
+                                              pull_data->content_mirrorlist,
+                                              deltapart_path, size,
+                                              OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                              pull_data->cancellable,
+                                              static_deltapart_fetch_on_complete,
+                                              fetch_data);
           pull_data->n_outstanding_deltapart_fetches++;
         }
     }

From c051ee4a35c90451f891eb52a0b44554df59912e Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 29 Dec 2016 15:57:53 -0500
Subject: [PATCH 10/27] build-sys: Minor makefile tweaks

I'm introducing a new binary in a later patch, and it makes
sense to move more things to be common into the common section.

Also I noticed we were missing an inclusion of common `$(AM_LDFLAGS)`, though
AFAIK this doesn't break anything right now.

Closes: #636
Approved by: jlebon
---
 Makefile-decls.am  |  1 +
 Makefile-ostree.am | 12 +++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Makefile-decls.am b/Makefile-decls.am
index e41586d1..06018594 100644
--- a/Makefile-decls.am
+++ b/Makefile-decls.am
@@ -30,6 +30,7 @@ sbin_PROGRAMS =
 bin_SCRIPTS =
 lib_LTLIBRARIES =
 libexec_PROGRAMS =
+pkglibexec_PROGRAMS =
 pkglibexec_SCRIPTS =
 noinst_LTLIBRARIES =
 noinst_PROGRAMS =
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index e6b1eabe..d46fc225 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -102,12 +102,14 @@ src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile
 EXTRA_DIST += src/ostree/parse-datetime.y
 CLEANFILES += src/ostree/parse-datetime.c
 
-ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(srcdir)/src/ostree \
-	$(NULL)
-ostree_bin_shared_ldadd = libglnx.la libbsdiff.la libotutil.la libostree-kernel-args.la libostree-1.la
+ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree \
+	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS)
+ostree_bin_shared_ldadd = $(AM_LDFLAGS) libglnx.la libotutil.la libostree-1.la \
+	$(OT_INTERNAL_GIO_UNIX_LIBS)
+
+ostree_CFLAGS = $(ostree_bin_shared_cflags)
+ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la $(LIBSYSTEMD_LIBS)
 
-ostree_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx
-ostree_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_GIO_UNIX_LIBS) $(LIBSYSTEMD_LIBS)
 
 if USE_LIBSOUP
 ostree_SOURCES += \

From ced22f6c9b096d6bb337886fafd06eae1b888005 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Fri, 30 Dec 2016 14:18:34 -0500
Subject: [PATCH 11/27] Split trivial-httpd into separate binary

Working on the libcurl backend, I hit the issue that the trivial-httpd program
depends on libsoup. I briefly considered having two versions, but libcurl is
client only, and moreover trivial-httpd is no longer trivial - it has various
features which are used by the test suite extensively.

Hence, what we'll do is build it as a separate binary which links to libsoup,
and use it during the tests. We *also* currently still provide `ostree
trivial-httpd` since some things use it like `rpm-ostree-toolbox` and the
Cockpit tests.

After those are ported to use some other webserver, I plan to add a build-time
option to drop it.

Closes: #636
Approved by: jlebon
---
 Makefile-ostree.am                    |  13 +-
 Makefile-tests.am                     |   1 +
 src/ostree/ostree-trivial-httpd.c     | 683 ++++++++++++++++++++++++++
 src/ostree/ot-builtin-trivial-httpd.c | 632 +-----------------------
 tests/libtest.sh                      |  10 +-
 tests/test-commit-sign.sh             |   2 +-
 tests/test-pull-contenturl.sh         |   3 +-
 tests/test-pull-metalink.sh           |   2 +-
 tests/test-pull-mirrorlist.sh         |   2 +-
 tests/test-pull-override-url.sh       |   2 +-
 10 files changed, 715 insertions(+), 635 deletions(-)
 create mode 100644 src/ostree/ostree-trivial-httpd.c

diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index d46fc225..05fec155 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -103,7 +103,8 @@ EXTRA_DIST += src/ostree/parse-datetime.y
 CLEANFILES += src/ostree/parse-datetime.c
 
 ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree \
-	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS)
+	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS) \
+	-DPKGLIBEXECDIR=\"$(pkglibexecdir)\"
 ostree_bin_shared_ldadd = $(AM_LDFLAGS) libglnx.la libotutil.la libostree-1.la \
 	$(OT_INTERNAL_GIO_UNIX_LIBS)
 
@@ -112,12 +113,14 @@ ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la
 
 
 if USE_LIBSOUP
-ostree_SOURCES += \
-	src/ostree/ot-builtin-pull.c \
-	src/ostree/ot-builtin-trivial-httpd.c \
-	$(NULL)
+ostree_SOURCES += src/ostree/ot-builtin-pull.c src/ostree/ot-builtin-trivial-httpd.c
 ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS)
 ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS)
+
+pkglibexec_PROGRAMS += ostree-trivial-httpd
+ostree_trivial_httpd_SOURCES = src/ostree/ostree-trivial-httpd.c
+ostree_trivial_httpd_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_SOUP_CFLAGS)
+ostree_trivial_httpd_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_SOUP_LIBS)
 endif
 
 if USE_LIBARCHIVE
diff --git a/Makefile-tests.am b/Makefile-tests.am
index 1f9cad48..63ceea1e 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -28,6 +28,7 @@ EXTRA_DIST += \
 # include the builddir in $PATH so we find our just-built ostree
 # binary.
 TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \
+	OSTREE_UNINSTALLED=$(abs_top_builddir) \
 	G_DEBUG=fatal-warnings \
 	GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd)$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH} \
 	LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd)$${LD_LIBRARY_PATH:+:$${LD_LIBRARY_PATH}} \
diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c
new file mode 100644
index 00000000..ef297af6
--- /dev/null
+++ b/src/ostree/ostree-trivial-httpd.c
@@ -0,0 +1,683 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2011,2013 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <libsoup/soup.h>
+
+#include <gio/gunixoutputstream.h>
+
+#include "ot-main.h"
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "otutil.h"
+
+#include <locale.h>
+#include <sys/socket.h>
+#include <sys/prctl.h>
+#include <signal.h>
+
+static char *opt_port_file = NULL;
+static char *opt_log = NULL;
+static gboolean opt_daemonize;
+static gboolean opt_autoexit;
+static gboolean opt_force_ranges;
+static int opt_random_500s_percentage;
+/* We have a strong upper bound for any unlikely
+ * cases involving repeated random 500s. */
+static int opt_random_500s_max = 100;
+static gint opt_port = 0;
+static gchar **opt_expected_cookies;
+static gchar **opt_expected_headers;
+
+static guint emitted_random_500s_count = 0;
+
+typedef struct {
+  int root_dfd;
+  gboolean running;
+  GOutputStream *log;
+} OtTrivialHttpd;
+
+static GOptionEntry options[] = {
+  { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &opt_daemonize, "Fork into background when ready", NULL },
+  { "autoexit", 0, 0, G_OPTION_ARG_NONE, &opt_autoexit, "Automatically exit when directory is deleted", NULL },
+  { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL },
+  { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" },
+  { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL },
+  { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" },
+  { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
+  { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
+  { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" },
+  { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" },
+  { NULL }
+};
+
+static void
+httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) __attribute__ ((format(printf, 2, 3)));
+
+static void
+httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...)
+{
+  g_autoptr(GString) str = NULL;
+  va_list args;
+  gsize written;
+
+  if (!httpd->log)
+    return;
+
+  {
+    g_autoptr(GDateTime) now = g_date_time_new_now_local ();
+    g_autofree char *timestamp = g_date_time_format (now, "%F %T");
+    str = g_string_new (timestamp);
+    g_string_append_printf (str, ".%06d - ", g_date_time_get_microsecond (now));
+  }
+
+  va_start (args, format);
+  g_string_append_vprintf (str, format, args);
+  va_end (args);
+
+  g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL);
+}
+
+static int
+compare_strings (gconstpointer a, gconstpointer b)
+{
+  const char **sa = (const char **)a;
+  const char **sb = (const char **)b;
+
+  return strcmp (*sa, *sb);
+}
+
+static GString *
+get_directory_listing (int dfd,
+                       const char *path)
+{
+  g_autoptr(GPtrArray) entries = g_ptr_array_new_with_free_func (g_free);
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  g_autoptr(GError) local_error = NULL;
+  GError **error = &local_error;
+  guint i;
+  char *escaped;
+  GString *listing;
+
+  listing = g_string_new ("<html>\r\n");
+
+  if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error))
+    goto out;
+
+  while (TRUE)
+    {
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, NULL, error))
+        goto out;
+
+      if (dent == NULL)
+        break;
+
+      escaped = g_markup_escape_text (dent->d_name, -1);
+      g_ptr_array_add (entries, escaped);
+    }
+
+  g_ptr_array_sort (entries, (GCompareFunc)compare_strings);
+
+  escaped = g_markup_escape_text (strchr (path, '/'), -1);
+  g_string_append_printf (listing, "<head><title>Index of %s</title></head>\r\n", escaped);
+  g_string_append_printf (listing, "<body><h1>Index of %s</h1>\r\n<p>\r\n", escaped);
+  g_free (escaped);
+  for (i = 0; i < entries->len; i++)
+    {
+      g_string_append_printf (listing, "<a href=\"%s\">%s</a><br>\r\n",
+                              (char *)entries->pdata[i], 
+                              (char *)entries->pdata[i]);
+      g_free (g_steal_pointer (&entries->pdata[i]));
+    }
+  g_string_append (listing, "</body>\r\n</html>\r\n");
+ out:
+  if (local_error)
+    g_printerr ("%s\n", local_error->message);
+  return listing;
+}
+
+/* Only allow reading files that have o+r, and for directories, o+x.
+ * This makes this server relatively safe to use on multiuser
+ * machines.
+ */
+static gboolean
+is_safe_to_access (struct stat *stbuf)
+{
+  /* Only regular files or directores */
+  if (!(S_ISREG (stbuf->st_mode) || S_ISDIR (stbuf->st_mode)))
+    return FALSE;
+  /* Must be o+r */
+  if (!(stbuf->st_mode & S_IROTH))
+    return FALSE;
+  /* For directories, must be o+x */
+  if (S_ISDIR (stbuf->st_mode) && !(stbuf->st_mode & S_IXOTH))
+    return FALSE;
+  return TRUE;
+}
+
+static void
+close_socket (SoupMessage *msg, gpointer user_data)
+{
+  SoupSocket *sock = user_data;
+  int sockfd;
+
+  /* Actually calling soup_socket_disconnect() here would cause
+   * us to leak memory, so just shutdown the socket instead.
+   */
+  sockfd = soup_socket_get_fd (sock);
+#ifdef G_OS_WIN32
+  shutdown (sockfd, SD_SEND);
+#else
+  shutdown (sockfd, SHUT_WR);
+#endif
+}
+
+static void
+do_get (OtTrivialHttpd    *self,
+        SoupServer        *server,
+        SoupMessage       *msg,
+        const char        *path,
+        SoupClientContext *context)
+{
+  char *slash;
+  int ret;
+  struct stat stbuf;
+
+  httpd_log (self, "serving %s\n", path);
+
+  if (opt_expected_cookies)
+    {
+      GSList *cookies = soup_cookies_from_request (msg);
+      GSList *l;
+      int i;
+
+      for (i = 0 ; opt_expected_cookies[i] != NULL; i++)
+        {
+          gboolean found = FALSE;
+          gchar *k = opt_expected_cookies[i];
+          gchar *v = strchr (k, '=') + 1;
+
+          for (l = cookies;  l != NULL ; l = g_slist_next (l))
+            {
+              SoupCookie *c = l->data;
+
+              if (!strncmp (k, soup_cookie_get_name (c), v - k - 1) &&
+                  !strcmp (v, soup_cookie_get_value (c)))
+                {
+                  found = TRUE;
+                  break;
+                }
+            }
+
+          if (!found)
+            {
+              httpd_log (self, "Expected cookie not found %s\n", k);
+              soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+              soup_cookies_free (cookies);
+              goto out;
+            }
+        }
+      soup_cookies_free (cookies);
+    }
+
+  if (opt_expected_headers)
+    {
+      for (int i = 0 ; opt_expected_headers[i] != NULL; i++)
+        {
+          const gchar *kv = opt_expected_headers[i];
+          const gchar *eq = strchr (kv, '=');
+
+          g_assert (eq);
+
+          {
+            g_autofree char *k = g_strndup (kv, eq - kv);
+            const gchar *expected_v = eq + 1;
+            const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k);
+
+            if (!found_v)
+              {
+                httpd_log (self, "Expected header not found %s\n", k);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+            if (strcmp (found_v, expected_v) != 0)
+              {
+                httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+          }
+        }
+    }
+
+  if (strstr (path, "../") != NULL)
+    {
+      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      goto out;
+    }
+
+  if (opt_random_500s_percentage > 0 &&
+      emitted_random_500s_count < opt_random_500s_max &&
+      g_random_int_range (0, 100) < opt_random_500s_percentage)
+    {
+      emitted_random_500s_count++;
+      soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+      goto out;
+    }
+
+  while (path[0] == '/')
+    path++;
+
+  do
+    ret = fstatat (self->root_dfd, path, &stbuf, 0);
+  while (ret == -1 && errno == EINTR);
+  if (ret == -1)
+    {
+      if (errno == EPERM)
+        soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      else if (errno == ENOENT)
+        soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND);
+      else
+        soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+      goto out;
+    }
+
+  if (!is_safe_to_access (&stbuf))
+    {
+      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      goto out;
+    }
+
+  if (S_ISDIR (stbuf.st_mode))
+    {
+      slash = strrchr (path, '/');
+      if (!slash || slash[1])
+        {
+          g_autofree char *redir_uri = NULL;
+
+          redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path);
+          soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY,
+                                     redir_uri);
+        }
+      else
+        {
+          g_autofree char *index_realpath = g_strconcat (path, "/index.html", NULL);
+          if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1)
+            {
+              g_autofree char *index_path = g_strconcat (path, "/index.html", NULL);
+              do_get (self, server, msg, index_path, context);
+            }
+          else
+            {
+              GString *listing = get_directory_listing (self->root_dfd, path);
+              soup_message_set_response (msg, "text/html",
+                                         SOUP_MEMORY_TAKE,
+                                         listing->str, listing->len);
+              soup_message_set_status (msg, SOUP_STATUS_OK);
+              g_string_free (listing, FALSE);
+            }
+        }
+    }
+  else 
+    {
+      if (!S_ISREG (stbuf.st_mode))
+        {
+          soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+          goto out;
+        }
+      
+      if (msg->method == SOUP_METHOD_GET)
+        {
+          glnx_fd_close int fd = -1;
+          g_autoptr(GMappedFile) mapping = NULL;
+          gsize buffer_length, file_size;
+          SoupRange *ranges;
+          int ranges_length;
+          gboolean have_ranges;
+
+          fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
+          if (fd < 0)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+              goto out;
+            }
+
+          mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
+          if (!mapping)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+              goto out;
+            }
+          (void) close (fd); fd = -1;
+
+          file_size = g_mapped_file_get_length (mapping);
+          have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length);
+          if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL)
+            {
+              SoupSocket *sock;
+              buffer_length = file_size/2;
+              soup_message_headers_set_content_length (msg->response_headers, file_size);
+              soup_message_headers_append (msg->response_headers,
+                                           "Connection", "close");
+
+              /* soup-message-io will wait for us to add
+               * another chunk after the first, to fill out
+               * the declared Content-Length. Instead, we
+               * forcibly close the socket at that point.
+               */
+              sock = soup_client_context_get_socket (context);
+              g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock);
+            }
+          else
+            buffer_length = file_size;
+
+          if (have_ranges)
+            {
+              if (ranges_length > 0 && ranges[0].start >= file_size)
+                {
+                  soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE);
+                  soup_message_headers_free_ranges (msg->request_headers, ranges);
+                  goto out;
+                }
+              soup_message_headers_free_ranges (msg->request_headers, ranges);
+            }
+          if (buffer_length > 0)
+            {
+              SoupBuffer *buffer;
+
+              buffer = soup_buffer_new_with_owner (g_mapped_file_get_contents (mapping),
+                                                   buffer_length,
+                                                   g_mapped_file_ref (mapping),
+                                                   (GDestroyNotify)g_mapped_file_unref);
+              soup_message_body_append_buffer (msg->response_body, buffer);
+              soup_buffer_free (buffer);
+            }
+        }
+      else /* msg->method == SOUP_METHOD_HEAD */
+        {
+          g_autofree char *length = NULL;
+
+          /* We could just use the same code for both GET and
+           * HEAD (soup-message-server-io.c will fix things up).
+           * But we'll optimize and avoid the extra I/O.
+           */
+          length = g_strdup_printf ("%lu", (gulong)stbuf.st_size);
+          soup_message_headers_append (msg->response_headers,
+                                       "Content-Length", length);
+        }
+      soup_message_set_status (msg, SOUP_STATUS_OK);
+    }
+ out:
+  {
+    guint status = 0;
+    g_autofree gchar *reason = NULL;
+
+    g_object_get (msg,
+                  "status-code", &status,
+                  "reason-phrase", &reason,
+                  NULL);
+    httpd_log (self, "  status: %s (%u)\n", reason, status);
+  }
+  return;
+}
+
+static void
+httpd_callback (SoupServer *server, SoupMessage *msg,
+                const char *path, GHashTable *query,
+                SoupClientContext *context, gpointer data)
+{
+  OtTrivialHttpd *self = data;
+
+  if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD)
+    do_get (self, server, msg, path, context);
+  else
+    soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED);
+}
+
+static void
+on_dir_changed (GFileMonitor  *mon,
+		GFile *file,
+		GFile *other,
+		GFileMonitorEvent  event,
+		gpointer user_data)
+{
+  OtTrivialHttpd *self = user_data;
+
+  if (event == G_FILE_MONITOR_EVENT_DELETED)
+    {
+      self->running = FALSE;
+      g_main_context_wakeup (NULL);
+    }
+}
+
+static gboolean
+run (int argc, char **argv, GCancellable *cancellable, GError **error)
+{
+  gboolean ret = FALSE;
+  g_autoptr(GOptionContext) context = NULL;
+  const char *dirpath;
+  OtTrivialHttpd appstruct = { 0, };
+  OtTrivialHttpd *app = &appstruct;
+  glnx_unref_object SoupServer *server = NULL;
+  g_autoptr(GFileMonitor) dirmon = NULL;
+
+  context = g_option_context_new ("[DIR] - Simple webserver");
+  g_option_context_add_main_entries (context, options, NULL);
+
+  app->root_dfd = -1;
+
+  if (!g_option_context_parse (context, &argc, &argv, error))
+    goto out;
+
+  if (argc > 1)
+    dirpath = argv[1];
+  else
+    dirpath = ".";
+
+  if (!glnx_opendirat (AT_FDCWD, dirpath, TRUE, &app->root_dfd, error))
+    goto out;
+
+  if (!(opt_random_500s_percentage >= 0 && opt_random_500s_percentage <= 99))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid --random-500s=%u", opt_random_500s_percentage);
+      goto out;
+    }
+
+  if (opt_log)
+    {
+      GOutputStream *stream = NULL;
+
+      if (g_strcmp0 (opt_log, "-") == 0)
+        {
+          if (opt_daemonize)
+            {
+              ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error);
+              goto out;
+            }
+          stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE));
+        }
+      else
+        {
+          g_autoptr(GFile) log_file;
+          GFileOutputStream* log_stream;
+
+          log_file = g_file_new_for_path (opt_log);
+          log_stream = g_file_create (log_file,
+                                      G_FILE_CREATE_PRIVATE,
+                                      cancellable,
+                                      error);
+          if (!log_stream)
+            goto out;
+          stream = G_OUTPUT_STREAM (log_stream);
+        }
+
+      app->log = stream;
+    }
+
+#if SOUP_CHECK_VERSION(2, 48, 0)
+  server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL);
+  if (!soup_server_listen_all (server, opt_port, 0, error))
+    goto out;
+#else
+  server = soup_server_new (SOUP_SERVER_PORT, opt_port,
+                            SOUP_SERVER_SERVER_HEADER, "ostree-httpd ",
+                            NULL);
+#endif
+
+  soup_server_add_handler (server, NULL, httpd_callback, app, NULL);
+  if (opt_port_file)
+    {
+      g_autofree char *portstr = NULL;
+#if SOUP_CHECK_VERSION(2, 48, 0)
+      GSList *listeners = soup_server_get_listeners (server);
+      g_autoptr(GSocket) listener = NULL;
+      g_autoptr(GSocketAddress) addr = NULL;
+      
+      g_assert (listeners);
+      listener = g_object_ref (listeners->data);
+      g_slist_free (listeners);
+      listeners = NULL;
+      addr = g_socket_get_local_address (listener, error);
+      if (!addr)
+        goto out;
+
+      g_assert (G_IS_INET_SOCKET_ADDRESS (addr));
+      
+      portstr = g_strdup_printf ("%u\n", g_inet_socket_address_get_port ((GInetSocketAddress*)addr));
+#else
+      portstr = g_strdup_printf ("%u\n", soup_server_get_port (server));
+#endif
+
+      if (g_strcmp0 ("-", opt_port_file) == 0)
+        {
+          fputs (portstr, stdout); // not g_print - this must go to stdout, not a handler
+          fflush (stdout);
+        }
+      else if (!g_file_set_contents (opt_port_file, portstr, strlen (portstr), error))
+        goto out;
+    }
+#if !SOUP_CHECK_VERSION(2, 48, 0)
+  soup_server_run_async (server);
+#endif
+  
+  if (opt_daemonize)
+    {
+      pid_t pid = fork();
+      if (pid == -1)
+        {
+          int errsv = errno;
+          g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (errsv),
+                               g_strerror (errsv));
+          goto out;
+        }
+      else if (pid > 0)
+        {
+          ret = TRUE;
+          goto out;
+        }
+      /* Child, continue */
+      /* Daemonising: close stdout/stderr so $() et al work on us */
+      fclose (stdout);
+      fclose (stdin);
+    }
+  else
+    {
+      /* Since we're used for testing purposes, let's just do this by
+       * default.  This ensures we exit when our parent does.
+       */
+      if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0)
+        {
+          if (errno != ENOSYS)
+            {
+              glnx_set_error_from_errno (error);
+              goto out;
+            }
+        }
+    }
+
+  app->running = TRUE;
+  if (opt_autoexit)
+    {
+      gboolean is_symlink = FALSE;
+      g_autoptr(GFile) root = NULL;
+      g_autoptr(GFileInfo) info = NULL;
+
+      root = g_file_new_for_path (dirpath);
+      info = g_file_query_info (root,
+                                G_FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,
+                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                cancellable, error);
+      if (!info)
+        goto out;
+
+      is_symlink = g_file_info_get_is_symlink (info);
+
+      if (is_symlink)
+        dirmon = g_file_monitor_file (root, 0, cancellable, error);
+      else
+        dirmon = g_file_monitor_directory (root, 0, cancellable, error);
+
+      if (!dirmon)
+        goto out;
+      g_signal_connect (dirmon, "changed", G_CALLBACK (on_dir_changed), app);
+    }
+  httpd_log (app, "serving at root %s\n", dirpath);
+  while (app->running)
+    g_main_context_iteration (NULL, TRUE);
+
+  ret = TRUE;
+ out:
+  if (app->root_dfd != -1)
+    (void) close (app->root_dfd);
+  g_clear_object (&app->log);
+  return ret;
+}
+
+int
+main (int    argc,
+      char **argv)
+{
+  g_autoptr(GError) error = NULL;
+  g_autoptr(GCancellable) cancellable = NULL;
+
+  setlocale (LC_ALL, "");
+
+  g_set_prgname (argv[0]);
+
+  if (!run (argc, argv, cancellable, &error))
+    {
+      int is_tty = isatty (1);
+      const char *prefix = "";
+      const char *suffix = "";
+      if (is_tty)
+        {
+          prefix = "\x1b[31m\x1b[1m"; /* red, bold */
+          suffix = "\x1b[22m\x1b[0m"; /* bold off, color reset */
+        }
+      g_printerr ("%serror: %s%s\n", prefix, suffix, error->message);
+      return 1;
+    }
+
+  return 0;
+}
diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c
index 0a553858..206970c5 100644
--- a/src/ostree/ot-builtin-trivial-httpd.c
+++ b/src/ostree/ot-builtin-trivial-httpd.c
@@ -1,6 +1,6 @@
 /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
  *
- * Copyright (C) 2011,2013 Colin Walters <walters@verbum.org>
+ * Copyright (C) 2016 Colin Walters <walters@verbum.org>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -20,634 +20,22 @@
 
 #include "config.h"
 
-#include <libsoup/soup.h>
-
-#include <gio/gunixoutputstream.h>
-
 #include "ot-main.h"
 #include "ot-builtins.h"
 #include "ostree.h"
 #include "otutil.h"
 
-#include <sys/socket.h>
-#include <sys/prctl.h>
-#include <signal.h>
-
-static char *opt_port_file = NULL;
-static char *opt_log = NULL;
-static gboolean opt_daemonize;
-static gboolean opt_autoexit;
-static gboolean opt_force_ranges;
-static int opt_random_500s_percentage;
-/* We have a strong upper bound for any unlikely
- * cases involving repeated random 500s. */
-static int opt_random_500s_max = 100;
-static gint opt_port = 0;
-static gchar **opt_expected_cookies;
-static gchar **opt_expected_headers;
-
-static guint emitted_random_500s_count = 0;
-
-typedef struct {
-  int root_dfd;
-  gboolean running;
-  GOutputStream *log;
-} OtTrivialHttpd;
-
-static GOptionEntry options[] = {
-  { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &opt_daemonize, "Fork into background when ready", NULL },
-  { "autoexit", 0, 0, G_OPTION_ARG_NONE, &opt_autoexit, "Automatically exit when directory is deleted", NULL },
-  { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL },
-  { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" },
-  { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL },
-  { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" },
-  { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
-  { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
-  { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" },
-  { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" },
-  { NULL }
-};
-
-static void
-httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) __attribute__ ((format(printf, 2, 3)));
-
-static void
-httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...)
-{
-  g_autoptr(GString) str = NULL;
-  va_list args;
-  gsize written;
-
-  if (!httpd->log)
-    return;
-
-  {
-    g_autoptr(GDateTime) now = g_date_time_new_now_local ();
-    g_autofree char *timestamp = g_date_time_format (now, "%F %T");
-    str = g_string_new (timestamp);
-    g_string_append_printf (str, ".%06d - ", g_date_time_get_microsecond (now));
-  }
-
-  va_start (args, format);
-  g_string_append_vprintf (str, format, args);
-  va_end (args);
-
-  g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL);
-}
-
-static int
-compare_strings (gconstpointer a, gconstpointer b)
-{
-  const char **sa = (const char **)a;
-  const char **sb = (const char **)b;
-
-  return strcmp (*sa, *sb);
-}
-
-static GString *
-get_directory_listing (int dfd,
-                       const char *path)
-{
-  g_autoptr(GPtrArray) entries = g_ptr_array_new_with_free_func (g_free);
-  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
-  g_autoptr(GError) local_error = NULL;
-  GError **error = &local_error;
-  guint i;
-  char *escaped;
-  GString *listing;
-
-  listing = g_string_new ("<html>\r\n");
-
-  if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error))
-    goto out;
-
-  while (TRUE)
-    {
-      struct dirent *dent;
-
-      if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, NULL, error))
-        goto out;
-
-      if (dent == NULL)
-        break;
-
-      escaped = g_markup_escape_text (dent->d_name, -1);
-      g_ptr_array_add (entries, escaped);
-    }
-
-  g_ptr_array_sort (entries, (GCompareFunc)compare_strings);
-
-  escaped = g_markup_escape_text (strchr (path, '/'), -1);
-  g_string_append_printf (listing, "<head><title>Index of %s</title></head>\r\n", escaped);
-  g_string_append_printf (listing, "<body><h1>Index of %s</h1>\r\n<p>\r\n", escaped);
-  g_free (escaped);
-  for (i = 0; i < entries->len; i++)
-    {
-      g_string_append_printf (listing, "<a href=\"%s\">%s</a><br>\r\n",
-                              (char *)entries->pdata[i], 
-                              (char *)entries->pdata[i]);
-      g_free (g_steal_pointer (&entries->pdata[i]));
-    }
-  g_string_append (listing, "</body>\r\n</html>\r\n");
- out:
-  if (local_error)
-    g_printerr ("%s\n", local_error->message);
-  return listing;
-}
-
-/* Only allow reading files that have o+r, and for directories, o+x.
- * This makes this server relatively safe to use on multiuser
- * machines.
- */
-static gboolean
-is_safe_to_access (struct stat *stbuf)
-{
-  /* Only regular files or directores */
-  if (!(S_ISREG (stbuf->st_mode) || S_ISDIR (stbuf->st_mode)))
-    return FALSE;
-  /* Must be o+r */
-  if (!(stbuf->st_mode & S_IROTH))
-    return FALSE;
-  /* For directories, must be o+x */
-  if (S_ISDIR (stbuf->st_mode) && !(stbuf->st_mode & S_IXOTH))
-    return FALSE;
-  return TRUE;
-}
-
-static void
-close_socket (SoupMessage *msg, gpointer user_data)
-{
-  SoupSocket *sock = user_data;
-  int sockfd;
-
-  /* Actually calling soup_socket_disconnect() here would cause
-   * us to leak memory, so just shutdown the socket instead.
-   */
-  sockfd = soup_socket_get_fd (sock);
-#ifdef G_OS_WIN32
-  shutdown (sockfd, SD_SEND);
-#else
-  shutdown (sockfd, SHUT_WR);
-#endif
-}
-
-static void
-do_get (OtTrivialHttpd    *self,
-        SoupServer        *server,
-        SoupMessage       *msg,
-        const char        *path,
-        SoupClientContext *context)
-{
-  char *slash;
-  int ret;
-  struct stat stbuf;
-
-  httpd_log (self, "serving %s\n", path);
-
-  if (opt_expected_cookies)
-    {
-      GSList *cookies = soup_cookies_from_request (msg);
-      GSList *l;
-      int i;
-
-      for (i = 0 ; opt_expected_cookies[i] != NULL; i++)
-        {
-          gboolean found = FALSE;
-          gchar *k = opt_expected_cookies[i];
-          gchar *v = strchr (k, '=') + 1;
-
-          for (l = cookies;  l != NULL ; l = g_slist_next (l))
-            {
-              SoupCookie *c = l->data;
-
-              if (!strncmp (k, soup_cookie_get_name (c), v - k - 1) &&
-                  !strcmp (v, soup_cookie_get_value (c)))
-                {
-                  found = TRUE;
-                  break;
-                }
-            }
-
-          if (!found)
-            {
-              httpd_log (self, "Expected cookie not found %s\n", k);
-              soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-              soup_cookies_free (cookies);
-              goto out;
-            }
-        }
-      soup_cookies_free (cookies);
-    }
-
-  if (opt_expected_headers)
-    {
-      for (int i = 0 ; opt_expected_headers[i] != NULL; i++)
-        {
-          const gchar *kv = opt_expected_headers[i];
-          const gchar *eq = strchr (kv, '=');
-
-          g_assert (eq);
-
-          {
-            g_autofree char *k = g_strndup (kv, eq - kv);
-            const gchar *expected_v = eq + 1;
-            const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k);
-
-            if (!found_v)
-              {
-                httpd_log (self, "Expected header not found %s\n", k);
-                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-                goto out;
-              }
-            if (strcmp (found_v, expected_v) != 0)
-              {
-                httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v);
-                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-                goto out;
-              }
-          }
-        }
-    }
-
-  if (strstr (path, "../") != NULL)
-    {
-      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      goto out;
-    }
-
-  if (opt_random_500s_percentage > 0 &&
-      emitted_random_500s_count < opt_random_500s_max &&
-      g_random_int_range (0, 100) < opt_random_500s_percentage)
-    {
-      emitted_random_500s_count++;
-      soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-      goto out;
-    }
-
-  while (path[0] == '/')
-    path++;
-
-  do
-    ret = fstatat (self->root_dfd, path, &stbuf, 0);
-  while (ret == -1 && errno == EINTR);
-  if (ret == -1)
-    {
-      if (errno == EPERM)
-        soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      else if (errno == ENOENT)
-        soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND);
-      else
-        soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-      goto out;
-    }
-
-  if (!is_safe_to_access (&stbuf))
-    {
-      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      goto out;
-    }
-
-  if (S_ISDIR (stbuf.st_mode))
-    {
-      slash = strrchr (path, '/');
-      if (!slash || slash[1])
-        {
-          g_autofree char *redir_uri = NULL;
-
-          redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path);
-          soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY,
-                                     redir_uri);
-        }
-      else
-        {
-          g_autofree char *index_realpath = g_strconcat (path, "/index.html", NULL);
-          if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1)
-            {
-              g_autofree char *index_path = g_strconcat (path, "/index.html", NULL);
-              do_get (self, server, msg, index_path, context);
-            }
-          else
-            {
-              GString *listing = get_directory_listing (self->root_dfd, path);
-              soup_message_set_response (msg, "text/html",
-                                         SOUP_MEMORY_TAKE,
-                                         listing->str, listing->len);
-              soup_message_set_status (msg, SOUP_STATUS_OK);
-              g_string_free (listing, FALSE);
-            }
-        }
-    }
-  else 
-    {
-      if (!S_ISREG (stbuf.st_mode))
-        {
-          soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-          goto out;
-        }
-      
-      if (msg->method == SOUP_METHOD_GET)
-        {
-          glnx_fd_close int fd = -1;
-          g_autoptr(GMappedFile) mapping = NULL;
-          gsize buffer_length, file_size;
-          SoupRange *ranges;
-          int ranges_length;
-          gboolean have_ranges;
-
-          fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
-          if (fd < 0)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-
-          mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
-          if (!mapping)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-          (void) close (fd); fd = -1;
-
-          file_size = g_mapped_file_get_length (mapping);
-          have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length);
-          if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL)
-            {
-              SoupSocket *sock;
-              buffer_length = file_size/2;
-              soup_message_headers_set_content_length (msg->response_headers, file_size);
-              soup_message_headers_append (msg->response_headers,
-                                           "Connection", "close");
-
-              /* soup-message-io will wait for us to add
-               * another chunk after the first, to fill out
-               * the declared Content-Length. Instead, we
-               * forcibly close the socket at that point.
-               */
-              sock = soup_client_context_get_socket (context);
-              g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock);
-            }
-          else
-            buffer_length = file_size;
-
-          if (have_ranges)
-            {
-              if (ranges_length > 0 && ranges[0].start >= file_size)
-                {
-                  soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE);
-                  soup_message_headers_free_ranges (msg->request_headers, ranges);
-                  goto out;
-                }
-              soup_message_headers_free_ranges (msg->request_headers, ranges);
-            }
-          if (buffer_length > 0)
-            {
-              SoupBuffer *buffer;
-
-              buffer = soup_buffer_new_with_owner (g_mapped_file_get_contents (mapping),
-                                                   buffer_length,
-                                                   g_mapped_file_ref (mapping),
-                                                   (GDestroyNotify)g_mapped_file_unref);
-              soup_message_body_append_buffer (msg->response_body, buffer);
-              soup_buffer_free (buffer);
-            }
-        }
-      else /* msg->method == SOUP_METHOD_HEAD */
-        {
-          g_autofree char *length = NULL;
-
-          /* We could just use the same code for both GET and
-           * HEAD (soup-message-server-io.c will fix things up).
-           * But we'll optimize and avoid the extra I/O.
-           */
-          length = g_strdup_printf ("%lu", (gulong)stbuf.st_size);
-          soup_message_headers_append (msg->response_headers,
-                                       "Content-Length", length);
-        }
-      soup_message_set_status (msg, SOUP_STATUS_OK);
-    }
- out:
-  {
-    guint status = 0;
-    g_autofree gchar *reason = NULL;
-
-    g_object_get (msg,
-                  "status-code", &status,
-                  "reason-phrase", &reason,
-                  NULL);
-    httpd_log (self, "  status: %s (%u)\n", reason, status);
-  }
-  return;
-}
-
-static void
-httpd_callback (SoupServer *server, SoupMessage *msg,
-                const char *path, GHashTable *query,
-                SoupClientContext *context, gpointer data)
-{
-  OtTrivialHttpd *self = data;
-
-  if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD)
-    do_get (self, server, msg, path, context);
-  else
-    soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED);
-}
-
-static void
-on_dir_changed (GFileMonitor  *mon,
-		GFile *file,
-		GFile *other,
-		GFileMonitorEvent  event,
-		gpointer user_data)
-{
-  OtTrivialHttpd *self = user_data;
-
-  if (event == G_FILE_MONITOR_EVENT_DELETED)
-    {
-      self->running = FALSE;
-      g_main_context_wakeup (NULL);
-    }
-}
-
 gboolean
 ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable, GError **error)
 {
-  gboolean ret = FALSE;
-  g_autoptr(GOptionContext) context = NULL;
-  const char *dirpath;
-  OtTrivialHttpd appstruct = { 0, };
-  OtTrivialHttpd *app = &appstruct;
-  glnx_unref_object SoupServer *server = NULL;
-  g_autoptr(GFileMonitor) dirmon = NULL;
+  g_autoptr(GPtrArray) new_argv = g_ptr_array_new ();
 
-  context = g_option_context_new ("[DIR] - Simple webserver");
-
-  app->root_dfd = -1;
-
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, error))
-    goto out;
-
-  if (argc > 1)
-    dirpath = argv[1];
-  else
-    dirpath = ".";
-
-  if (!glnx_opendirat (AT_FDCWD, dirpath, TRUE, &app->root_dfd, error))
-    goto out;
-
-  if (!(opt_random_500s_percentage >= 0 && opt_random_500s_percentage <= 99))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Invalid --random-500s=%u", opt_random_500s_percentage);
-      goto out;
-    }
-
-  if (opt_log)
-    {
-      GOutputStream *stream = NULL;
-
-      if (g_strcmp0 (opt_log, "-") == 0)
-        {
-          if (opt_daemonize)
-            {
-              ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error);
-              goto out;
-            }
-          stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE));
-        }
-      else
-        {
-          g_autoptr(GFile) log_file;
-          GFileOutputStream* log_stream;
-
-          log_file = g_file_new_for_path (opt_log);
-          log_stream = g_file_create (log_file,
-                                      G_FILE_CREATE_PRIVATE,
-                                      cancellable,
-                                      error);
-          if (!log_stream)
-            goto out;
-          stream = G_OUTPUT_STREAM (log_stream);
-        }
-
-      app->log = stream;
-    }
-
-#if SOUP_CHECK_VERSION(2, 48, 0)
-  server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL);
-  if (!soup_server_listen_all (server, opt_port, 0, error))
-    goto out;
-#else
-  server = soup_server_new (SOUP_SERVER_PORT, opt_port,
-                            SOUP_SERVER_SERVER_HEADER, "ostree-httpd ",
-                            NULL);
-#endif
-
-  soup_server_add_handler (server, NULL, httpd_callback, app, NULL);
-  if (opt_port_file)
-    {
-      g_autofree char *portstr = NULL;
-#if SOUP_CHECK_VERSION(2, 48, 0)
-      GSList *listeners = soup_server_get_listeners (server);
-      g_autoptr(GSocket) listener = NULL;
-      g_autoptr(GSocketAddress) addr = NULL;
-      
-      g_assert (listeners);
-      listener = g_object_ref (listeners->data);
-      g_slist_free (listeners);
-      listeners = NULL;
-      addr = g_socket_get_local_address (listener, error);
-      if (!addr)
-        goto out;
-
-      g_assert (G_IS_INET_SOCKET_ADDRESS (addr));
-      
-      portstr = g_strdup_printf ("%u\n", g_inet_socket_address_get_port ((GInetSocketAddress*)addr));
-#else
-      portstr = g_strdup_printf ("%u\n", soup_server_get_port (server));
-#endif
-
-      if (g_strcmp0 ("-", opt_port_file) == 0)
-        {
-          fputs (portstr, stdout); // not g_print - this must go to stdout, not a handler
-          fflush (stdout);
-        }
-      else if (!g_file_set_contents (opt_port_file, portstr, strlen (portstr), error))
-        goto out;
-    }
-#if !SOUP_CHECK_VERSION(2, 48, 0)
-  soup_server_run_async (server);
-#endif
-  
-  if (opt_daemonize)
-    {
-      pid_t pid = fork();
-      if (pid == -1)
-        {
-          int errsv = errno;
-          g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (errsv),
-                               g_strerror (errsv));
-          goto out;
-        }
-      else if (pid > 0)
-        {
-          ret = TRUE;
-          goto out;
-        }
-      /* Child, continue */
-      /* Daemonising: close stdout/stderr so $() et al work on us */
-      fclose (stdout);
-      fclose (stdin);
-    }
-  else
-    {
-      /* Since we're used for testing purposes, let's just do this by
-       * default.  This ensures we exit when our parent does.
-       */
-      if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0)
-        {
-          if (errno != ENOSYS)
-            {
-              glnx_set_error_from_errno (error);
-              goto out;
-            }
-        }
-    }
-
-  app->running = TRUE;
-  if (opt_autoexit)
-    {
-      gboolean is_symlink = FALSE;
-      g_autoptr(GFile) root = NULL;
-      g_autoptr(GFileInfo) info = NULL;
-
-      root = g_file_new_for_path (dirpath);
-      info = g_file_query_info (root,
-                                G_FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,
-                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                cancellable, error);
-      if (!info)
-        goto out;
-
-      is_symlink = g_file_info_get_is_symlink (info);
-
-      if (is_symlink)
-        dirmon = g_file_monitor_file (root, 0, cancellable, error);
-      else
-        dirmon = g_file_monitor_directory (root, 0, cancellable, error);
-
-      if (!dirmon)
-        goto out;
-      g_signal_connect (dirmon, "changed", G_CALLBACK (on_dir_changed), app);
-    }
-  httpd_log (app, "serving at root %s\n", dirpath);
-  while (app->running)
-    g_main_context_iteration (NULL, TRUE);
-
-  ret = TRUE;
- out:
-  if (app->root_dfd != -1)
-    (void) close (app->root_dfd);
-  g_clear_object (&app->log);
-  return ret;
+  g_ptr_array_add (new_argv, PKGLIBEXECDIR "/ostree-trivial-httpd");
+  for (int i = 1; i < argc; i++)
+    g_ptr_array_add (new_argv, argv[i]);
+  g_ptr_array_add (new_argv, NULL);
+  execvp (new_argv->pdata[0], (char**)new_argv->pdata);
+  /* Fall through on error */
+  glnx_set_error_from_errno (error);
+  return FALSE;
 }
diff --git a/tests/libtest.sh b/tests/libtest.sh
index c0bf8d0d..137d9534 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -102,6 +102,12 @@ else
     fi
 fi
 
+if test -n "${OSTREE_UNINSTALLED:-}"; then
+    OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd
+else
+    OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd"
+fi
+
 assert_streq () {
     test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1)
 }
@@ -257,7 +263,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 
@@ -379,7 +385,7 @@ EOF
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir} ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port
+    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 
diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh
index 60265c1a..f963b104 100755
--- a/tests/test-commit-sign.sh
+++ b/tests/test-commit-sign.sh
@@ -53,7 +53,7 @@ cd ${test_tmpdir}
 mkdir ${test_tmpdir}/httpd
 cd httpd
 ln -s ${test_tmpdir}/ostree-srv ostree
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
 port=$(cat ${test_tmpdir}/httpd-port)
 assert_streq $port 18081
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
diff --git a/tests/test-pull-contenturl.sh b/tests/test-pull-contenturl.sh
index 16dcbe4f..d74d619b 100755
--- a/tests/test-pull-contenturl.sh
+++ b/tests/test-pull-contenturl.sh
@@ -51,8 +51,7 @@ fi
 find ${test_tmpdir}/ostree-srv/gnomerepo/objects \
   ! -name '*.commitmeta' -type f | xargs rm
 
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
-  -p ${test_tmpdir}/httpd-content-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-content-port
 content_port=$(cat ${test_tmpdir}/httpd-content-port)
 echo "http://127.0.0.1:${content_port}" > ${test_tmpdir}/httpd-content-address
 
diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh
index 2a1a73e7..07d619df 100755
--- a/tests/test-pull-metalink.sh
+++ b/tests/test-pull-metalink.sh
@@ -29,7 +29,7 @@ echo '1..9'
 cd ${test_tmpdir}
 mkdir metalink-data
 cd metalink-data
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
 metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port)
 echo "http://127.0.0.1:${metalink_port}" > ${test_tmpdir}/metalink-httpd-address
 
diff --git a/tests/test-pull-mirrorlist.sh b/tests/test-pull-mirrorlist.sh
index 454014ca..13f40e7a 100755
--- a/tests/test-pull-mirrorlist.sh
+++ b/tests/test-pull-mirrorlist.sh
@@ -33,7 +33,7 @@ setup_mirror () {
   cd $name
   cp -a ${test_tmpdir}/ostree-srv ostree
 
-  ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
+  ${OSTREE_HTTPD} --autoexit --daemonize \
     -p ${test_tmpdir}/${name}-port
   port=$(cat ${test_tmpdir}/${name}-port)
   echo "http://127.0.0.1:${port}" > ${test_tmpdir}/${name}-address
diff --git a/tests/test-pull-override-url.sh b/tests/test-pull-override-url.sh
index d81b3454..928013a5 100755
--- a/tests/test-pull-override-url.sh
+++ b/tests/test-pull-override-url.sh
@@ -50,7 +50,7 @@ mkdir mirror-httpd
 cd mirror-httpd
 ln -s ${test_tmpdir}/mirror-srv ostree
 mirror_log="${test_tmpdir}/mirror_log"
-${CMD_PREFIX} ostree trivial-httpd --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
+${OSTREE_HTTPD} --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
 port=$(cat ${test_tmpdir}/mirror-httpd-port)
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/mirror-httpd-address
 

From d9f43cd2fbbd48b34c06552205eaf27eed1e1ae0 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 10:45:58 -0500
Subject: [PATCH 12/27] pull: Rework delta superblock fetches to be async

For the pending libcurl port, the backend is a bit more
sensitive to the main context setup.  The delta superblock
fetch here is a synchronous request in the section that's
supposed to be async.

Now, libsoup definitely supports mixing sync and async requests, but it wasn't
hard to help the libcurl port here by making this one async. Now fetchers are
either sync or async.

Closes: #636
Approved by: jlebon
---
 src/libostree/ostree-repo-pull.c | 218 +++++++++++++++++--------------
 1 file changed, 120 insertions(+), 98 deletions(-)

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 92efd592..54a3a922 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -67,6 +67,7 @@ typedef struct {
   gint          n_scanned_metadata;
 
   gboolean          gpg_verify;
+  gboolean          require_static_deltas;
   gboolean          gpg_verify_summary;
   gboolean          has_tombstone_commits;
 
@@ -178,6 +179,10 @@ update_progress (gpointer user_data)
   if (! pull_data->progress)
     return FALSE;
 
+  /* In dry run, we only emit progress once metadata is done */
+  if (pull_data->dry_run && pull_data->n_outstanding_metadata_fetches > 0)
+    return TRUE;
+
   outstanding_writes = pull_data->n_outstanding_content_write_requests +
     pull_data->n_outstanding_metadata_write_requests +
     pull_data->n_outstanding_deltapart_write_requests;
@@ -1416,75 +1421,6 @@ load_remote_repo_config (OtPullData    *pull_data,
   return ret;
 }
 
-static gboolean
-request_static_delta_superblock_sync (OtPullData  *pull_data,
-                                      const char  *from_revision,
-                                      const char  *to_revision,
-                                      GVariant   **out_delta_superblock,
-                                      GCancellable *cancellable,
-                                      GError     **error)
-{
-  gboolean ret = FALSE;
-  g_autoptr(GVariant) ret_delta_superblock = NULL;
-  g_autofree char *delta_name =
-    _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision);
-  g_autoptr(GBytes) delta_superblock_data = NULL;
-
-  if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
-                                                   pull_data->content_mirrorlist,
-                                                   delta_name, FALSE, TRUE,
-                                                   &delta_superblock_data,
-                                                   OSTREE_MAX_METADATA_SIZE,
-                                                   pull_data->cancellable, error))
-    goto out;
-
-  if (delta_superblock_data)
-    {
-      {
-        g_autofree gchar *delta = NULL;
-        g_autofree guchar *ret_csum = NULL;
-        guchar *summary_csum;
-        g_autoptr (GInputStream) summary_is = NULL;
-
-        summary_is = g_memory_input_stream_new_from_data (g_bytes_get_data (delta_superblock_data, NULL),
-                                                          g_bytes_get_size (delta_superblock_data),
-                                                          NULL);
-
-        if (!ot_gio_checksum_stream (summary_is, &ret_csum, cancellable, error))
-          goto out;
-
-        delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
-        summary_csum = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
-
-        /* At this point we've GPG verified the data, so in theory
-         * could trust that they provided the right data, but let's
-         * make this a hard error.
-         */
-        if (pull_data->gpg_verify_summary && !summary_csum)
-          {
-            g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                         "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
-            goto out;
-          }
-
-        if (summary_csum && memcmp (summary_csum, ret_csum, 32))
-          {
-            g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
-            goto out;
-          }
-      }
-
-      ret_delta_superblock = g_variant_ref_sink (g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
-                                                                           delta_superblock_data, FALSE));
-    }
-  
-  ret = TRUE;
-  if (out_delta_superblock)
-    *out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
- out:
-  return ret;
-}
-
 static gboolean
 process_one_static_delta_fallback (OtPullData   *pull_data,
                                    gboolean      delta_byteswap,
@@ -1749,6 +1685,100 @@ process_one_static_delta (OtPullData   *pull_data,
   return ret;
 }
 
+typedef struct {
+  OtPullData *pull_data;
+  char *from_revision;
+  char *to_revision;
+} FetchDeltaSuperData;
+
+static void
+on_superblock_fetched (GObject   *src,
+                       GAsyncResult *res,
+                       gpointer      data)
+
+{
+  FetchDeltaSuperData *fdata = data;
+  OtPullData *pull_data = fdata->pull_data;
+  GError *local_error = NULL;
+  GError **error = &local_error;
+  g_autoptr(GBytes) delta_superblock_data = NULL;
+  const char *from_revision = fdata->from_revision;
+  const char *to_revision = fdata->to_revision;
+
+  if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
+                                                 res,
+                                                 &delta_superblock_data,
+                                                 error))
+    {
+      if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+        goto out;
+
+      g_clear_error (&local_error);
+
+      if (pull_data->require_static_deltas)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Static deltas required, but none found for %s to %s",
+                       from_revision, to_revision);
+          goto out;
+        }
+
+      queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
+    }
+  else
+    {
+      g_autofree gchar *delta = NULL;
+      g_autofree guchar *ret_csum = NULL;
+      guchar *summary_csum;
+      g_autoptr (GInputStream) summary_is = NULL;
+      g_autoptr(GVariant) delta_superblock = NULL;
+
+      summary_is = g_memory_input_stream_new_from_data (g_bytes_get_data (delta_superblock_data, NULL),
+                                                        g_bytes_get_size (delta_superblock_data),
+                                                        NULL);
+
+      if (!ot_gio_checksum_stream (summary_is, &ret_csum, pull_data->cancellable, error))
+        goto out;
+
+      delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
+      summary_csum = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
+
+      /* At this point we've GPG verified the data, so in theory
+       * could trust that they provided the right data, but let's
+       * make this a hard error.
+       */
+      if (pull_data->gpg_verify_summary && !summary_csum)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
+          goto out;
+        }
+
+      if (summary_csum && memcmp (summary_csum, ret_csum, 32))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
+          goto out;
+        }
+
+      delta_superblock = g_variant_ref_sink (g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
+                                                                       delta_superblock_data, FALSE));
+
+      g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock));
+      if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock,
+                                     pull_data->cancellable, error))
+        goto out;
+    }
+
+ out:
+  g_free (fdata->from_revision);
+  g_free (fdata->to_revision);
+  g_free (fdata);
+  g_assert (pull_data->n_outstanding_metadata_fetches > 0);
+  pull_data->n_outstanding_metadata_fetches--;
+  pull_data->n_fetched_metadata++;
+  check_outstanding_requests_handle_error (pull_data, local_error);
+}
+
 static gboolean
 validate_variant_is_csum (GVariant       *csum,
                           GError        **error)
@@ -2344,7 +2374,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_autofree char **override_commit_ids = NULL;
   GSource *update_timeout = NULL;
   gboolean disable_static_deltas = FALSE;
-  gboolean require_static_deltas = FALSE;
   gboolean opt_gpg_verify_set = FALSE;
   gboolean opt_gpg_verify_summary_set = FALSE;
   const char *url_override = NULL;
@@ -2368,7 +2397,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary);
       (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth);
       (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
-      (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
+      (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas);
       (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
       (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
       (void) g_variant_lookup (options, "override-url", "&s", &url_override);
@@ -2386,11 +2415,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   for (i = 0; dirs_to_pull != NULL && dirs_to_pull[i] != NULL; i++)
     g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE);
 
-  g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
+  g_return_val_if_fail (!(disable_static_deltas && pull_data->require_static_deltas), FALSE);
   /* We only do dry runs with static deltas, because we don't really have any
    * in-advance information for bare fetches.
    */
-  g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);
+  g_return_val_if_fail (!pull_data->dry_run || pull_data->require_static_deltas, FALSE);
 
   pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
   pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@@ -2655,7 +2684,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   /* For local pulls, default to disabling static deltas so that the
    * exact object files are copied.
    */
-  if (pull_data->remote_repo_local && !require_static_deltas)
+  if (pull_data->remote_repo_local && !pull_data->require_static_deltas)
     disable_static_deltas = TRUE;
 
   pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -2710,7 +2739,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         goto out;
       }
 
-    if (!bytes_summary && require_static_deltas)
+    if (!bytes_summary && pull_data->require_static_deltas)
       {
         g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                      "Fetch configured to require static deltas, but no summary found");
@@ -2939,7 +2968,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       g_autofree char *from_revision = NULL;
       const char *ref = key;
       const char *to_revision = value;
-      g_autoptr(GVariant) delta_superblock = NULL;
 
       if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
                                     &from_revision, error))
@@ -2948,31 +2976,25 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       if (!(disable_static_deltas || mirroring_into_archive || pull_data->is_commit_only) &&
           (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0))
         {
-          if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
-                                                     &delta_superblock, cancellable, error))
-            goto out;
-        }
-          
-      if (!delta_superblock)
-        {
-          if (require_static_deltas)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Static deltas required, but none found for %s to %s",
-                           from_revision, to_revision);
-              goto out;
-            }
-          g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
-          queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
+          g_autofree char *delta_name =
+            _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision);
+          FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1);
+          fdata->pull_data = pull_data;
+          fdata->from_revision = g_strdup (from_revision);
+          fdata->to_revision = g_strdup (to_revision);
+
+          _ostree_fetcher_request_to_membuf (pull_data->fetcher,
+                                             pull_data->content_mirrorlist,
+                                             delta_name, 0,
+                                             OSTREE_MAX_METADATA_SIZE,
+                                             0, pull_data->cancellable,
+                                             on_superblock_fetched, fdata);
+          pull_data->n_outstanding_metadata_fetches++;
+          pull_data->n_requested_metadata++;
         }
       else
         {
-          g_debug ("processing delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
-          g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock));
-          if (!process_one_static_delta (pull_data, from_revision, to_revision,
-                                         delta_superblock,
-                                         cancellable, error))
-            goto out;
+          queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
         }
     }
 

From 2a71afc50779f2b0cd13ea48762af348c396ad2b Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 12 Jan 2017 10:02:23 -0500
Subject: [PATCH 13/27] trivial-httpd: Daemonize better

I was trying to debug `test-pull-c`, and typing `Ctrl-C` in gdb
ended up sending `SIGINT` to trivial-httpd as well, killing it.

Daemonize a bit more properly to avoid this. I also followed the standard
`/dev/null` guidelines.

Closes: #643
Approved by: jlebon
---
 src/ostree/ostree-trivial-httpd.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c
index ef297af6..ddcb2fc0 100644
--- a/src/ostree/ostree-trivial-httpd.c
+++ b/src/ostree/ostree-trivial-httpd.c
@@ -30,6 +30,7 @@
 #include "otutil.h"
 
 #include <locale.h>
+#include <err.h>
 #include <sys/socket.h>
 #include <sys/prctl.h>
 #include <signal.h>
@@ -597,9 +598,12 @@ run (int argc, char **argv, GCancellable *cancellable, GError **error)
           goto out;
         }
       /* Child, continue */
+      if (setsid () < 0)
+        err (1, "setsid");
       /* Daemonising: close stdout/stderr so $() et al work on us */
-      fclose (stdout);
-      fclose (stdin);
+      freopen("/dev/null", "r", stdin);
+      freopen("/dev/null", "w", stdout);
+      freopen("/dev/null", "w", stderr);
     }
   else
     {

From 9a77017d87b74c5e2895cdd64ad098018929403f Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 12 Jan 2017 10:53:57 -0500
Subject: [PATCH 14/27] .dir-locals.el: Standard Emacs indentation config

Better than having it per-file.

Closes: #644
Approved by: jlebon
---
 .dir-locals.el | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .dir-locals.el

diff --git a/.dir-locals.el b/.dir-locals.el
new file mode 100644
index 00000000..35140408
--- /dev/null
+++ b/.dir-locals.el
@@ -0,0 +1 @@
+((c-mode . ((indent-tabs-mode . nil) (c-file-style . "gnu"))))

From 5782e0a1d3cb525324bf34b4a811b5802a750b9f Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Mon, 16 Jan 2017 13:41:20 -0500
Subject: [PATCH 15/27] unlock: Fix description for --hotfix

Came up during an IRC discussion.  The text is accurate but
not very helpful.

Closes: #647
Approved by: dustymabe
---
 src/ostree/ot-admin-builtin-unlock.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ostree/ot-admin-builtin-unlock.c b/src/ostree/ot-admin-builtin-unlock.c
index 0f22d0a6..aecba51b 100644
--- a/src/ostree/ot-admin-builtin-unlock.c
+++ b/src/ostree/ot-admin-builtin-unlock.c
@@ -34,7 +34,7 @@
 static gboolean opt_hotfix;
 
 static GOptionEntry options[] = {
-  { "hotfix", 0, 0, G_OPTION_ARG_NONE, &opt_hotfix, "Keep the current deployment as default", NULL },
+  { "hotfix", 0, 0, G_OPTION_ARG_NONE, &opt_hotfix, "Retain changes across reboots", NULL },
   { NULL }
 };
 

From 2f71136eec7fa6326fb66147664960eb822a066f Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 17 Jan 2017 10:16:31 -0500
Subject: [PATCH 16/27] tests: Alias assert_not_reached() -> fatal()

We had a lot of copies of the "echo something 1>&2; exit 1" code even though
`assert_not_reached()` was it.  Hence, I think we need a shorter alias for that.

Doing this particularly since I noticed a missing `1` in an `exit 1` call in the
rpm-ostree copy of this.

Closes: #648
Approved by: jlebon
---
 tests/basic-test.sh |  4 ++--
 tests/libtest.sh    | 33 ++++++++++++++++-----------------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/tests/basic-test.sh b/tests/basic-test.sh
index bb387501..0547c89d 100755
--- a/tests/basic-test.sh
+++ b/tests/basic-test.sh
@@ -26,7 +26,7 @@ echo "ok checkout"
 
 $OSTREE rev-parse test2
 $OSTREE rev-parse 'test2^'
-$OSTREE rev-parse 'test2^^' 2>/dev/null && (echo 1>&2 "rev-parse test2^^ unexpectedly succeeded!"; exit 1)
+$OSTREE rev-parse 'test2^^' 2>/dev/null && fatal "rev-parse test2^^ unexpectedly succeeded!"
 echo "ok rev-parse"
 
 checksum=$($OSTREE rev-parse test2)
@@ -294,7 +294,7 @@ rm repo3/refs/heads/* repo3/refs/remotes/* -rf
 ${CMD_PREFIX} ostree --repo=repo3 prune --refs-only
 find repo3/objects -name '*.commit' > objlist-after-prune
 if cmp -s objlist-before-prune objlist-after-prune; then
-    echo "Prune didn't delete anything!"; exit 1
+    fatal "Prune didn't delete anything!"
 fi
 rm repo3 objlist-before-prune objlist-after-prune -rf
 echo "ok prune"
diff --git a/tests/libtest.sh b/tests/libtest.sh
index 137d9534..db57128a 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -29,9 +29,13 @@ else
   test_builddir=$(dirname $0)
 fi
 
-assert_not_reached () {
+fatal() {
     echo $@ 1>&2; exit 1
 }
+# fatal() is shorter to type, but retain this alias
+assert_not_reached () {
+    fatal "$@"
+}
 
 test_tmpdir=$(pwd)
 
@@ -109,54 +113,51 @@ else
 fi
 
 assert_streq () {
-    test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1)
+    test "$1" = "$2" || fatal "$1 != $2"
 }
 
 assert_str_match () {
     if ! echo "$1" | grep -E -q "$2"; then
-	(echo 1>&2 "$1 does not match regexp $2"; exit 1)
+	      fatal "$1 does not match regexp $2"
     fi
 }
 
 assert_not_streq () {
-    (! test "$1" = "$2") || (echo 1>&2 "$1 == $2"; exit 1)
+    (! test "$1" = "$2") || fatal "$1 == $2"
 }
 
 assert_has_file () {
-    test -f "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1)
+    test -f "$1" || fatal "Couldn't find '$1'"
 }
 
 assert_has_dir () {
-    test -d "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1)
+    test -d "$1" || fatal "Couldn't find '$1'"
 }
 
 assert_not_has_file () {
     if test -f "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' exists"
-        exit 1
+        fatal "File '$1' exists"
     fi
 }
 
 assert_not_file_has_content () {
     if grep -q -e "$2" "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' incorrectly matches regexp '$2'"
-        exit 1
+        fatal "File '$1' incorrectly matches regexp '$2'"
     fi
 }
 
 assert_not_has_dir () {
     if test -d "$1"; then
-	echo 1>&2 "Directory '$1' exists"; exit 1
+	      fatal "Directory '$1' exists"
     fi
 }
 
 assert_file_has_content () {
     if ! grep -q -e "$2" "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' doesn't match regexp '$2'"
-        exit 1
+        fatal "File '$1' doesn't match regexp '$2'"
     fi
 }
 
@@ -175,8 +176,7 @@ assert_symlink_has_content () {
 assert_file_empty() {
     if test -s "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' is not empty"
-        exit 1
+        fatal "File '$1' is not empty"
     fi
 }
 
@@ -184,8 +184,7 @@ assert_files_hardlinked() {
     f1=$(stat -c %i $1)
     f2=$(stat -c %i $2)
     if [ "$f1" != "$f2" ]; then
-        echo 1>&2 "Files '$1' and '$2' are not hardlinked"
-        exit 1
+        fatal "Files '$1' and '$2' are not hardlinked"
     fi
 }
 

From b28b785f01b364baac709e87f99fe2ce20c7914e Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 18 Jan 2017 20:56:28 -0500
Subject: [PATCH 17/27] pull: Fix theoretical checksum collision for metadata
 fetches

I was making some other changes in this code, and noticed that we were adding
checksums without object types into the same hash table for metadata. We should
*never* do this with both metadata content objects, since in theory a content
object could have the same hash as metadata.

I don't actually think it's possible in practice for pure metadata to collide,
since they have different structures, but let's do this anyways since it's
conceptually right.

Closes: #651
Approved by: giuseppe
---
 src/libostree/ostree-repo-pull.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 54a3a922..3dabb346 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -80,7 +80,7 @@ typedef struct {
   GHashTable       *commit_to_depth; /* Maps commit checksum maximum depth */
   GHashTable       *scanned_metadata; /* Maps object name to itself */
   GHashTable       *requested_metadata; /* Maps object name to itself */
-  GHashTable       *requested_content; /* Maps object name to itself */
+  GHashTable       *requested_content; /* Maps checksum to itself */
   guint             n_outstanding_metadata_fetches;
   guint             n_outstanding_metadata_write_requests;
   guint             n_outstanding_content_fetches;
@@ -1267,7 +1267,7 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
   if (g_hash_table_lookup (pull_data->scanned_metadata, object))
     return TRUE;
 
-  is_requested = g_hash_table_lookup (pull_data->requested_metadata, tmp_checksum) != NULL;
+  is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL;
   if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
                                cancellable, error))
     goto out;
@@ -1292,10 +1292,9 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
 
   if (!is_stored && !is_requested)
     {
-      char *duped_checksum = g_strdup (tmp_checksum);
       gboolean do_fetch_detached;
 
-      g_hash_table_add (pull_data->requested_metadata, duped_checksum);
+      g_hash_table_add (pull_data->requested_metadata, g_variant_ref (object));
 
       do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
       enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, do_fetch_detached, FALSE);
@@ -1467,10 +1466,11 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
     { 
       if (OSTREE_OBJECT_TYPE_IS_META (objtype))
         {
-          if (!g_hash_table_lookup (pull_data->requested_metadata, checksum))
+          g_autoptr(GVariant) objname = ostree_object_name_serialize (checksum, objtype);
+          if (!g_hash_table_lookup (pull_data->requested_metadata, objname))
             {
               gboolean do_fetch_detached;
-              g_hash_table_add (pull_data->requested_metadata, checksum);
+              g_hash_table_add (pull_data->requested_metadata, g_variant_ref (objname));
               
               do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
               enqueue_one_object_request (pull_data, checksum, objtype, NULL, do_fetch_detached, FALSE);
@@ -2451,8 +2451,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                        (GDestroyNotify)g_variant_unref, NULL);
   pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                         (GDestroyNotify)g_free, NULL);
-  pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
-                                                         (GDestroyNotify)g_free, NULL);
+  pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+                                                         (GDestroyNotify)g_variant_unref, NULL);
   if (dir_to_pull != NULL || dirs_to_pull != NULL)
     {
       pull_data->dirs = g_ptr_array_new_with_free_func (g_free);

From e6952de3d67f8b9896cb15ffa31f50b4e0758714 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Mon, 9 Jan 2017 10:02:19 -0500
Subject: [PATCH 18/27] fetcher: Rework API to use strings for tls keys/db

This is prep for the libcurl porting. `GTlsCertificate/GTlsDatabase` are
abstract classes implemented in glib-networking for gnutls. curl's APIs take
file paths as strings, so it's easier to work on both if we move the GLib TLS
bits into the libsoup code.

Closes: #651
Approved by: giuseppe
---
 src/libostree/ostree-fetcher.c              | 66 +++++++++++++--------
 src/libostree/ostree-fetcher.h              |  5 +-
 src/libostree/ostree-repo-pull.c            | 20 +------
 src/libostree/ostree-tls-cert-interaction.c | 16 ++++-
 src/libostree/ostree-tls-cert-interaction.h |  3 +-
 5 files changed, 61 insertions(+), 49 deletions(-)

diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index cee33186..a178abfe 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -52,6 +52,7 @@ typedef struct {
   SoupSession *session;  /* not referenced */
   GMainContext *main_context;
   volatile gint running;
+  GError *initialization_error; /* Any failure to load the db */
 
   int tmpdir_dfd;
   char *tmpdir_name;
@@ -357,12 +358,14 @@ static void
 session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure,
                                        gpointer data)
 {
-  GTlsCertificate *cert = data;
+  const char *cert_and_key_path = data; /* str\0str\0 in one malloc buf */
+  const char *cert_path = cert_and_key_path;
+  const char *key_path = cert_and_key_path + strlen (cert_and_key_path) + 1;
   glnx_unref_object OstreeTlsCertInteraction *interaction = NULL;
 
   /* The GTlsInteraction instance must be created in the
    * session thread so it uses the correct GMainContext. */
-  interaction = _ostree_tls_cert_interaction_new (cert);
+  interaction = _ostree_tls_cert_interaction_new (cert_path, key_path);
 
   g_object_set (thread_closure->session,
                 SOUP_SESSION_TLS_INTERACTION,
@@ -374,13 +377,19 @@ static void
 session_thread_set_tls_database_cb (ThreadClosure *thread_closure,
                                     gpointer data)
 {
-  GTlsDatabase *database = data;
+  const char *db_path = data;
 
-  if (database != NULL)
+  if (db_path != NULL)
     {
-      g_object_set (thread_closure->session,
-                    SOUP_SESSION_TLS_DATABASE,
-                    database, NULL);
+      glnx_unref_object GTlsDatabase *tlsdb = NULL;
+
+      g_clear_error (&thread_closure->initialization_error);
+      tlsdb = g_tls_file_database_new (db_path, &thread_closure->initialization_error);
+
+      if (tlsdb)
+        g_object_set (thread_closure->session,
+                      SOUP_SESSION_TLS_DATABASE,
+                      tlsdb, NULL);
     }
   else
     {
@@ -452,6 +461,13 @@ session_thread_request_uri (ThreadClosure *thread_closure,
   pending = g_task_get_task_data (task);
   cancellable = g_task_get_cancellable (task);
 
+  /* If we caught an error in init, re-throw it for every request */
+  if (thread_closure->initialization_error)
+    {
+      g_task_return_error (task, g_error_copy (thread_closure->initialization_error));
+      return;
+    }
+
   create_pending_soup_request (pending, &local_error);
   if (local_error != NULL)
     {
@@ -797,16 +813,24 @@ _ostree_fetcher_set_cookie_jar (OstreeFetcher *self,
 
 void
 _ostree_fetcher_set_client_cert (OstreeFetcher   *self,
-                                 GTlsCertificate *cert)
+                                 const char      *cert_path,
+                                 const char      *key_path)
 {
+  g_autoptr(GString) buf = NULL;
   g_return_if_fail (OSTREE_IS_FETCHER (self));
-  g_return_if_fail (G_IS_TLS_CERTIFICATE (cert));
+
+  if (cert_path)
+    {
+      buf = g_string_new (cert_path);
+      g_string_append_c (buf, '\0');
+      g_string_append (buf, key_path);
+    }
 
 #ifdef HAVE_LIBSOUP_CLIENT_CERTS
   session_thread_idle_add (self->thread_closure,
                            session_thread_set_tls_interaction_cb,
-                           g_object_ref (cert),
-                           (GDestroyNotify) g_object_unref);
+                           g_string_free (g_steal_pointer (&buf), FALSE),
+                           (GDestroyNotify) g_free);
 #else
   g_warning ("This version of OSTree is compiled without client side certificate support");
 #endif
@@ -814,24 +838,14 @@ _ostree_fetcher_set_client_cert (OstreeFetcher   *self,
 
 void
 _ostree_fetcher_set_tls_database (OstreeFetcher *self,
-                                  GTlsDatabase  *db)
+                                  const char    *tlsdb_path)
 {
   g_return_if_fail (OSTREE_IS_FETCHER (self));
-  g_return_if_fail (db == NULL || G_IS_TLS_DATABASE (db));
 
-  if (db != NULL)
-    {
-      session_thread_idle_add (self->thread_closure,
-                               session_thread_set_tls_database_cb,
-                               g_object_ref (db),
-                               (GDestroyNotify) g_object_unref);
-    }
-  else
-    {
-      session_thread_idle_add (self->thread_closure,
-                               session_thread_set_tls_database_cb,
-                               NULL, (GDestroyNotify) NULL);
-    }
+  session_thread_idle_add (self->thread_closure,
+                           session_thread_set_tls_database_cb,
+                           g_strdup (tlsdb_path),
+                           (GDestroyNotify) g_free);
 }
 
 void
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index a57907e4..f19eb73b 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -93,10 +93,11 @@ void _ostree_fetcher_set_proxy (OstreeFetcher *fetcher,
                                 const char    *proxy);
 
 void _ostree_fetcher_set_client_cert (OstreeFetcher *fetcher,
-                                     GTlsCertificate *cert);
+                                      const char     *cert_path,
+                                      const char     *key_path);
 
 void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
-                                       GTlsDatabase *db);
+                                       const char    *tlsdb_path);
 
 void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
                                         GVariant      *extra_headers);
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 3dabb346..45995a30 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -1951,17 +1951,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
       }
     else if (tls_client_cert_path != NULL)
       {
-        g_autoptr(GTlsCertificate) client_cert = NULL;
-
-        g_assert (tls_client_key_path != NULL);
-
-        client_cert = g_tls_certificate_new_from_files (tls_client_cert_path,
-                                                        tls_client_key_path,
-                                                        error);
-        if (client_cert == NULL)
-          goto out;
-
-        _ostree_fetcher_set_client_cert (fetcher, client_cert);
+        _ostree_fetcher_set_client_cert (fetcher, tls_client_cert_path, tls_client_key_path);
       }
   }
 
@@ -1975,13 +1965,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
 
     if (tls_ca_path != NULL)
       {
-        g_autoptr(GTlsDatabase) db = NULL;
-
-        db = g_tls_file_database_new (tls_ca_path, error);
-        if (db == NULL)
-          goto out;
-
-        _ostree_fetcher_set_tls_database (fetcher, db);
+        _ostree_fetcher_set_tls_database (fetcher, tls_ca_path);
       }
   }
 
diff --git a/src/libostree/ostree-tls-cert-interaction.c b/src/libostree/ostree-tls-cert-interaction.c
index 846d5725..7e60f9de 100644
--- a/src/libostree/ostree-tls-cert-interaction.c
+++ b/src/libostree/ostree-tls-cert-interaction.c
@@ -24,6 +24,8 @@ struct _OstreeTlsCertInteraction
 {
   GTlsInteraction parent_instance;
 
+  char *cert_path;
+  char *key_path;
   GTlsCertificate *cert;
 };
 
@@ -44,6 +46,14 @@ request_certificate (GTlsInteraction              *interaction,
                      GError                      **error)
 {
   OstreeTlsCertInteraction *self = (OstreeTlsCertInteraction*)interaction;
+
+  if (!self->cert)
+    {
+      self->cert = g_tls_certificate_new_from_files (self->cert_path, self->key_path, error);
+      if (!self->cert)
+        return G_TLS_INTERACTION_FAILED;
+    }
+
   g_tls_connection_set_certificate (connection, self->cert);
   return G_TLS_INTERACTION_HANDLED;
 }
@@ -61,9 +71,11 @@ _ostree_tls_cert_interaction_class_init (OstreeTlsCertInteractionClass *klass)
 }
 
 OstreeTlsCertInteraction *
-_ostree_tls_cert_interaction_new (GTlsCertificate *cert)
+_ostree_tls_cert_interaction_new (const char *cert_path,
+                                  const char *key_path)
 {
   OstreeTlsCertInteraction *self = g_object_new (OSTREE_TYPE_TLS_CERT_INTERACTION, NULL);
-  self->cert = g_object_ref (cert);
+  self->cert_path = g_strdup (cert_path);
+  self->key_path = g_strdup (key_path);
   return self;
 }
diff --git a/src/libostree/ostree-tls-cert-interaction.h b/src/libostree/ostree-tls-cert-interaction.h
index c81097c5..ae4532f7 100644
--- a/src/libostree/ostree-tls-cert-interaction.h
+++ b/src/libostree/ostree-tls-cert-interaction.h
@@ -34,6 +34,7 @@ typedef struct _OstreeTlsCertInteractionClass   OstreeTlsCertInteractionClass;
 
 GType                       _ostree_tls_cert_interaction_get_type    (void) G_GNUC_CONST;
 
-OstreeTlsCertInteraction *  _ostree_tls_cert_interaction_new         (GTlsCertificate *cert);
+OstreeTlsCertInteraction *  _ostree_tls_cert_interaction_new         (const char *cert_path,
+                                                                      const char *key_path);
 
 G_END_DECLS

From 56891f9d485ee064be3f8d40f64243e87b395da1 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Sun, 8 Jan 2017 20:44:20 -0500
Subject: [PATCH 19/27] tests: Don't inject newline in URL

It turns out libsoup strips all whitespace even *inside* a URL. We could do that
for libcurl too but...really, people shouldn't do that. In this test we were
adding the trailing newline into the URL. If someone complains who is using the
libcurl code we can deal with it then.

Closes: #651
Approved by: giuseppe
---
 tests/test-pull-c.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test-pull-c.c b/tests/test-pull-c.c
index 43d5d61d..c2d64dcc 100644
--- a/tests/test-pull-c.c
+++ b/tests/test-pull-c.c
@@ -49,6 +49,8 @@ test_data_init (TestData *td)
   if (!g_file_get_contents ("httpd-address", &http_address, NULL, error))
     goto out;
 
+  g_strstrip (http_address);
+
   repo_url = g_strconcat (http_address, "/ostree/gnomerepo", NULL);
 
   { g_autoptr(GVariantBuilder) builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));

From 686f91062fd29f776710c5a176a0f5c62cc49af4 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Sun, 8 Jan 2017 20:45:58 -0500
Subject: [PATCH 20/27] tests: Loosen error regexp

libcurl AFAICS doesn't have an API to convert HTTP code :arrow_right: error
string, so let's make the test regexp operate on both.

Closes: #651
Approved by: giuseppe
---
 tests/test-pull-repeated.sh | 2 +-
 tests/test-pull-resume.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test-pull-repeated.sh b/tests/test-pull-repeated.sh
index 5a3af81c..8934e430 100755
--- a/tests/test-pull-repeated.sh
+++ b/tests/test-pull-repeated.sh
@@ -34,7 +34,7 @@ for x in $(seq 200); do
 	echo "Success on iteration ${x}"
 	break;
     fi
-    assert_file_has_content err.txt "500.*Internal Server Error"
+    assert_file_has_content err.txt "\(500.*Internal Server Error\)\|\(HTTP 500\)"
 done
 
 ${CMD_PREFIX} ostree --repo=repo fsck
diff --git a/tests/test-pull-resume.sh b/tests/test-pull-resume.sh
index 1e7220d0..06cd0793 100755
--- a/tests/test-pull-resume.sh
+++ b/tests/test-pull-resume.sh
@@ -42,7 +42,7 @@ do
   if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.log; then
     break
   fi
-  assert_file_has_content err.log 'error:.*Download incomplete'
+  assert_file_has_content err.log 'error:.*\(Download incomplete\)\|\(Transferred a partial file\)'
 done
 if ${CMD_PREFIX} ostree --repo=repo fsck; then
     echo "ok, pull succeeded!"

From b260fa764cd0a3328c58341e5313b6fba8944d6d Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Fri, 13 Jan 2017 12:29:17 -0500
Subject: [PATCH 21/27] libtest: Enable web server logs

Now that we're daemonizing, it's useful to have the logs.  I
wanted this while debugging cookies.

Closes: #651
Approved by: giuseppe
---
 tests/libtest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/libtest.sh b/tests/libtest.sh
index db57128a..927b8b1a 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -262,7 +262,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 

From 01fb30b839ad27f638ca65b0a8d3cdd2b2002e66 Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@debian.org>
Date: Thu, 19 Jan 2017 11:21:58 +0000
Subject: [PATCH 22/27] Fix TAP syntax in test-basic-user.sh, and run it

In its initial commit, Alexander Larsson wrote

    This works standalone, but unfortunately it breaks in
    gnome-desktop-testing-runner as /tmp doesn't support
    xattrs, so it is not installed atm.

but we now (a) use /var/tmp, and (b) explicitly skip the test if
xattr support is unavailable. So it should be OK to run now.

Closes: #652
Approved by: cgwalters
---
 Makefile-tests.am        | 2 +-
 tests/test-basic-user.sh | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/Makefile-tests.am b/Makefile-tests.am
index 63ceea1e..f37c59da 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -45,6 +45,7 @@ dist_uninstalled_test_scripts = tests/test-symbols.sh
 
 dist_test_scripts = \
 	tests/test-basic.sh \
+	tests/test-basic-user.sh \
 	tests/test-pull-subpath.sh \
 	tests/test-archivez.sh \
 	tests/test-remote-add.sh \
@@ -119,7 +120,6 @@ dist_installed_test_data = tests/archive-test.sh \
 	tests/pull-test.sh \
 	tests/admin-test.sh \
 	tests/basic-test.sh \
-	tests/test-basic-user.sh \
 	tests/corrupt-repo-ref.js \
 	tests/pre-endian-deltas-repo-big.tar.xz \
 	tests/pre-endian-deltas-repo-little.tar.xz \
diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh
index 42e6a864..3e11545e 100755
--- a/tests/test-basic-user.sh
+++ b/tests/test-basic-user.sh
@@ -23,9 +23,6 @@ set -euo pipefail
 
 skip_without_user_xattrs
 
-echo "1..1"
-
 setup_test_repository "bare-user"
-echo "ok setup"
 
 . $(dirname $0)/basic-test.sh

From 9a3f82caae03f4abdbdc80c491c0b55a27e6a06e Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@debian.org>
Date: Thu, 19 Jan 2017 11:23:30 +0000
Subject: [PATCH 23/27] Sourced test snippets: remove shebang and make
 non-executable

They are installed non-executable, which makes Debian's Lintian
packaging consistency check complain that #! is only useful
in executable scripts. But in fact they are not useful to execute
directly (they rely on setup being done in the script that sources
them), so just chmod them -x.

Closes: #652
Approved by: cgwalters
---
 tests/admin-test.sh   | 3 ++-
 tests/archive-test.sh | 4 ++--
 tests/basic-test.sh   | 4 ++--
 tests/pull-test.sh    | 4 ++--
 4 files changed, 8 insertions(+), 7 deletions(-)
 mode change 100755 => 100644 tests/admin-test.sh
 mode change 100755 => 100644 tests/archive-test.sh
 mode change 100755 => 100644 tests/basic-test.sh
 mode change 100755 => 100644 tests/pull-test.sh

diff --git a/tests/admin-test.sh b/tests/admin-test.sh
old mode 100755
new mode 100644
index 76fc8b85..cc06fe6f
--- a/tests/admin-test.sh
+++ b/tests/admin-test.sh
@@ -1,4 +1,5 @@
-#!/bin/bash
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011,2014 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
diff --git a/tests/archive-test.sh b/tests/archive-test.sh
old mode 100755
new mode 100644
index e6f67cf5..9c5f8a47
--- a/tests/archive-test.sh
+++ b/tests/archive-test.sh
@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
diff --git a/tests/basic-test.sh b/tests/basic-test.sh
old mode 100755
new mode 100644
index 0547c89d..5cad8ab3
--- a/tests/basic-test.sh
+++ b/tests/basic-test.sh
@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
old mode 100755
new mode 100644
index 56258b6c..56b24a0c
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or

From 9d94fc40c80ab01b3f96d37dced970a07e8323af Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@debian.org>
Date: Thu, 19 Jan 2017 11:34:57 +0000
Subject: [PATCH 24/27] Make corrupt-repo-ref.js executable

Debian's Lintian packaging consistency check complains that it isn't
executable but has a #! line. In fact it's reasonable to run this
script directly, so make it executable, and put it in a _scripts
variable so it will be installed executable.

Closes: #652
Approved by: cgwalters
---
 Makefile-tests.am         | 8 +++++---
 tests/corrupt-repo-ref.js | 0
 2 files changed, 5 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 tests/corrupt-repo-ref.js

diff --git a/Makefile-tests.am b/Makefile-tests.am
index f37c59da..957b15a8 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -120,15 +120,17 @@ dist_installed_test_data = tests/archive-test.sh \
 	tests/pull-test.sh \
 	tests/admin-test.sh \
 	tests/basic-test.sh \
-	tests/corrupt-repo-ref.js \
 	tests/pre-endian-deltas-repo-big.tar.xz \
 	tests/pre-endian-deltas-repo-little.tar.xz \
 	$(NULL)
 
 EXTRA_DIST += tests/libtest.sh 
 
-dist_test_extra_scripts = tests/bootloader-entries-crosscheck.py \
-     tests/ostree-grub-generator
+dist_test_extra_scripts = \
+	tests/bootloader-entries-crosscheck.py \
+	tests/corrupt-repo-ref.js \
+	tests/ostree-grub-generator \
+	$(NULL)
 
 # We can't use nobase_ as we need to strip off the tests/, can't
 # use plain installed_ as we do need the gpghome/ prefix.
diff --git a/tests/corrupt-repo-ref.js b/tests/corrupt-repo-ref.js
old mode 100644
new mode 100755

From 5c940987e768523ef1411b65bcaad09fba6befef Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 1 Dec 2016 09:28:24 -0500
Subject: [PATCH 25/27] Add support for more selective pruning

There are use cases for having a single repo with branches
with different lifecycles; a simple example of what I was
trying to do in CentOS Atomic Host work is have "stable"
and "devel" branches, were we want to prune devel, but
retain *all* of stable.

This patch is split into two parts - first we add a low level "delete all
objects not in this set" API, and change the current prune API
to use this.

Next, we move more logic into the "ostree prune" command. This paves the way for
demonstrating how more sophisticated algorithms/logic could be developed outside
of the ostree core.

Also, the --keep-younger-than logic already lived in the commandline, so it
makes sense to keep extending it there.

Closes: https://github.com/ostreedev/ostree/issues/604

Closes: #646
Approved by: jlebon
---
 apidoc/ostree-sections.txt        |   1 +
 src/libostree/libostree.sym       |   6 +-
 src/libostree/ostree-repo-prune.c | 164 ++++++++++++++------
 src/libostree/ostree-repo.h       |  21 +++
 src/ostree/ot-builtin-prune.c     | 240 ++++++++++++++++++++----------
 tests/test-prune.sh               |  67 ++++++++-
 6 files changed, 366 insertions(+), 133 deletions(-)

diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt
index 3cda9150..dbd12526 100644
--- a/apidoc/ostree-sections.txt
+++ b/apidoc/ostree-sections.txt
@@ -368,6 +368,7 @@ ostree_repo_commit_traverse_iter_next
 OstreeRepoPruneFlags
 ostree_repo_prune
 ostree_repo_prune_static_deltas
+ostree_repo_prune_from_reachable
 OstreeRepoPullFlags
 ostree_repo_pull
 ostree_repo_pull_one_dir
diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym
index 0b8410fb..888a99ba 100644
--- a/src/libostree/libostree.sym
+++ b/src/libostree/libostree.sym
@@ -371,12 +371,10 @@ global:
  *                         NOTE NOTE NOTE
  */
 
-/* Remove comment when first new symbol is added
-LIBOSTREE_2016.XX {
+LIBOSTREE_2017.1 {
 global:
-	someostree_symbol_deleteme;
+        ostree_repo_prune_from_reachable;
 } LIBOSTREE_2016.14;
-*/
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste
diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c
index 22bff915..1b2a8257 100644
--- a/src/libostree/ostree-repo-prune.c
+++ b/src/libostree/ostree-repo-prune.c
@@ -254,6 +254,61 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit,
   return ret;
 }
 
+static gboolean
+repo_prune_internal (OstreeRepo        *self,
+                     GHashTable        *objects,
+                     OstreeRepoPruneOptions *options,
+                     gint              *out_objects_total,
+                     gint              *out_objects_pruned,
+                     guint64           *out_pruned_object_size_total,
+                     GCancellable      *cancellable,
+                     GError           **error)
+{
+  gboolean ret = FALSE;
+  GHashTableIter hash_iter;
+  gpointer key, value;
+  OtPruneData data = { 0, };
+
+  data.repo = self;
+  data.reachable = g_hash_table_ref (options->reachable);
+
+  g_hash_table_iter_init (&hash_iter, objects);
+  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+    {
+      GVariant *serialized_key = key;
+      GVariant *objdata = value;
+      const char *checksum;
+      OstreeObjectType objtype;
+      gboolean is_loose;
+
+      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+      g_variant_get_child (objdata, 0, "b", &is_loose);
+
+      if (!is_loose)
+        continue;
+
+      if (!maybe_prune_loose_object (&data, options->flags, checksum, objtype,
+                                     cancellable, error))
+        goto out;
+    }
+
+  if (!ostree_repo_prune_static_deltas (self, NULL, cancellable, error))
+    goto out;
+
+  if (!_ostree_repo_prune_tmp (self, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+  *out_objects_total = (data.n_reachable_meta + data.n_unreachable_meta +
+                        data.n_reachable_content + data.n_unreachable_content);
+  *out_objects_pruned = (data.n_unreachable_meta + data.n_unreachable_content);
+  *out_pruned_object_size_total = data.freed_bytes;
+ out:
+  if (data.reachable)
+    g_hash_table_unref (data.reachable);
+  return ret;
+}
+
 /**
  * ostree_repo_prune:
  * @self: Repo
@@ -289,39 +344,42 @@ ostree_repo_prune (OstreeRepo        *self,
                    GCancellable      *cancellable,
                    GError           **error)
 {
-  gboolean ret = FALSE;
   GHashTableIter hash_iter;
   gpointer key, value;
   g_autoptr(GHashTable) objects = NULL;
   g_autoptr(GHashTable) all_refs = NULL;
-  OtPruneData data = { 0, };
+  g_autoptr(GHashTable) reachable = NULL;
   gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
 
-  data.repo = self;
-  data.reachable = ostree_repo_traverse_new_reachable ();
+  reachable = ostree_repo_traverse_new_reachable ();
+
+  /* This original prune API has fixed logic for traversing refs or all commits
+   * combined with actually deleting content. The newer backend API just does
+   * the deletion.
+   */
 
   if (refs_only)
     {
       if (!ostree_repo_list_refs (self, NULL, &all_refs,
                                   cancellable, error))
-        goto out;
-      
+        return FALSE;
+
       g_hash_table_iter_init (&hash_iter, all_refs);
-      
+
       while (g_hash_table_iter_next (&hash_iter, &key, &value))
         {
           const char *checksum = value;
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable,
+          if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
                                                   cancellable, error))
-            goto out;
+            return FALSE;
         }
     }
 
   if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,
                                  &objects, cancellable, error))
-    goto out;
+    return FALSE;
 
   if (!refs_only)
     {
@@ -338,45 +396,57 @@ ostree_repo_prune (OstreeRepo        *self,
             continue;
 
           g_debug ("Finding objects to keep for commit %s", checksum);
-          if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable,
+          if (!ostree_repo_traverse_commit_union (self, checksum, depth, reachable,
                                                   cancellable, error))
-            goto out;
+            return FALSE;
         }
     }
 
-  g_hash_table_iter_init (&hash_iter, objects);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      GVariant *objdata = value;
-      const char *checksum;
-      OstreeObjectType objtype;
-      gboolean is_loose;
-      
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-      g_variant_get_child (objdata, 0, "b", &is_loose);
-
-      if (!is_loose)
-        continue;
-
-      if (!maybe_prune_loose_object (&data, flags, checksum, objtype,
-                                     cancellable, error))
-        goto out;
-    }
-
-  if (!ostree_repo_prune_static_deltas (self, NULL, cancellable, error))
-    goto out;
-
-  if (!_ostree_repo_prune_tmp (self, cancellable, error))
-    goto out;
-
-  ret = TRUE;
-  *out_objects_total = (data.n_reachable_meta + data.n_unreachable_meta +
-                        data.n_reachable_content + data.n_unreachable_content);
-  *out_objects_pruned = (data.n_unreachable_meta + data.n_unreachable_content);
-  *out_pruned_object_size_total = data.freed_bytes;
- out:
-  if (data.reachable)
-    g_hash_table_unref (data.reachable);
-  return ret;
+  { OstreeRepoPruneOptions opts = { flags, reachable };
+    return repo_prune_internal (self, objects, &opts,
+                                out_objects_total, out_objects_pruned,
+                                out_pruned_object_size_total, cancellable, error);
+  }
+}
+
+/**
+ * ostree_repo_prune_from_reachable:
+ * @self: Repo
+ * @options: Options controlling prune process
+ * @out_objects_total: (out): Number of objects found
+ * @out_objects_pruned: (out): Number of objects deleted
+ * @out_pruned_object_size_total: (out): Storage size in bytes of objects deleted
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Delete content from the repository.  This function is the "backend"
+ * half of the higher level ostree_repo_prune().  To use this function,
+ * you determine the root set yourself, and this function finds all other
+ * unreferenced objects and deletes them.
+ *
+ * Use this API when you want to perform more selective pruning - for example,
+ * retain all commits from a production branch, but just GC some history from
+ * your dev branch.
+ *
+ * The %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE flag may be specified to just determine
+ * statistics on objects that would be deleted, without actually deleting them.
+ */
+gboolean
+ostree_repo_prune_from_reachable (OstreeRepo        *self,
+                                  OstreeRepoPruneOptions *options,
+                                  gint              *out_objects_total,
+                                  gint              *out_objects_pruned,
+                                  guint64           *out_pruned_object_size_total,
+                                  GCancellable      *cancellable,
+                                  GError           **error)
+{
+  g_autoptr(GHashTable) objects = NULL;
+
+  if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,
+                                 &objects, cancellable, error))
+    return FALSE;
+
+  return repo_prune_internal (self, objects, options, out_objects_total,
+                              out_objects_pruned, out_pruned_object_size_total,
+                              cancellable, error);
 }
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index d5303e41..341a4d9c 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -970,6 +970,27 @@ gboolean ostree_repo_prune (OstreeRepo        *self,
                             GCancellable      *cancellable,
                             GError           **error);
 
+struct _OstreeRepoPruneOptions {
+  OstreeRepoPruneFlags flags;
+
+  GHashTable *reachable; /* Set<GVariant> (object names) */
+
+  gboolean unused_bools[6];
+  int unused_ints[6];
+  gpointer unused_ptrs[7];
+};
+
+typedef struct _OstreeRepoPruneOptions OstreeRepoPruneOptions;
+
+_OSTREE_PUBLIC
+gboolean ostree_repo_prune_from_reachable (OstreeRepo             *self,
+                                           OstreeRepoPruneOptions *options,
+                                           gint              *out_objects_total,
+                                           gint              *out_objects_pruned,
+                                           guint64           *out_pruned_object_size_total,
+                                           GCancellable           *cancellable,
+                                           GError              **error);
+
 /**
  * OstreeRepoPullFlags:
  * @OSTREE_REPO_PULL_FLAGS_NONE: No special options for pull
diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c
index d226c84b..853c051f 100644
--- a/src/ostree/ot-builtin-prune.c
+++ b/src/ostree/ot-builtin-prune.c
@@ -34,6 +34,7 @@ static gint opt_depth = -1;
 static gboolean opt_refs_only;
 static char *opt_delete_commit;
 static char *opt_keep_younger_than;
+static char **opt_retain_branch_depth;
 
 static GOptionEntry options[] = {
   { "no-prune", 0, 0, G_OPTION_ARG_NONE, &opt_no_prune, "Only display unreachable objects; don't delete", NULL },
@@ -42,6 +43,7 @@ static GOptionEntry options[] = {
   { "delete-commit", 0, 0, G_OPTION_ARG_STRING, &opt_delete_commit, "Specify a commit to delete", "COMMIT" },
   { "keep-younger-than", 0, 0, G_OPTION_ARG_STRING, &opt_keep_younger_than, "Prune all commits older than the specified date", "DATE" },
   { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" },
+  { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" },
   { NULL }
 };
 
@@ -82,87 +84,53 @@ delete_commit (OstreeRepo *repo, const char *commit_to_delete, GCancellable *can
 }
 
 static gboolean
-prune_commits_keep_younger_than_date (OstreeRepo *repo, const char *date, GCancellable *cancellable, GError **error)
+traverse_keep_younger_than (OstreeRepo *repo, const char *checksum,
+                            struct timespec *ts,
+                            GHashTable *reachable,
+                            GCancellable *cancellable, GError **error)
 {
-  g_autoptr(GHashTable) refs = NULL;
-  g_autoptr(GHashTable) ref_heads = g_hash_table_new (g_str_hash, g_str_equal);
-  g_autoptr(GHashTable) objects = NULL;
-  GHashTableIter hash_iter;
-  gpointer key, value;
-  struct timespec ts;
-  gboolean ret = FALSE;
+  g_autofree char *next_checksum = g_strdup (checksum);
+  g_autoptr(GVariant) commit = NULL;
 
-  if (!parse_datetime (&ts, date, NULL))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Could not parse '%s'", date);
-      goto out;
-    }
-
-  if (!ot_enable_tombstone_commits (repo, error))
-    goto out;
-
-  if (!ostree_repo_list_refs (repo, NULL, &refs, cancellable, error))
-    goto out;
-
-  /* We used to prune the HEAD of a given ref by default, but that's
-   * broken for a few reasons.  One is that people may use branches as
-   * tags.  Second is that if we do it, we should be deleting the ref
-   * too, otherwise e.g. `summary -u` breaks trying to load it, etc.
+  /* This is the first commit in our loop, which has a ref pointing to it. We
+   * don't want to auto-prune it.
    */
-  g_hash_table_iter_init (&hash_iter, refs);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+  if (!ostree_repo_traverse_commit_union (repo, checksum, 0, reachable,
+                                          cancellable, error))
+    return FALSE;
+
+  while (TRUE)
     {
-      /* Value is lifecycle bound to refs */
-      g_hash_table_add (ref_heads, (char*)value);
-    }
-
-  if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
-                                 cancellable, error))
-    goto out;
-
-  g_hash_table_iter_init (&hash_iter, objects);
-
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      const char *checksum;
-      OstreeObjectType objtype;
       guint64 commit_timestamp;
-      g_autoptr(GVariant) commit = NULL;
 
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+      if (!ostree_repo_load_variant_if_exists (repo, OSTREE_OBJECT_TYPE_COMMIT, next_checksum,
+                                               &commit, error))
+        return FALSE;
 
-      if (objtype != OSTREE_OBJECT_TYPE_COMMIT)
-        continue;
-
-      if (g_hash_table_contains (ref_heads, checksum))
-        continue;
-
-      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
-                                     &commit, error))
-        goto out;
+      if (!commit)
+        break; /* This commit was pruned, so we're done */
 
       commit_timestamp = ostree_commit_get_timestamp (commit);
-      if (commit_timestamp < ts.tv_sec)
+      /* Is this commit newer than our --keep-younger-than spec? */
+      if (commit_timestamp >= ts->tv_sec)
         {
-          if (opt_static_deltas_only)
-            {
-              if(!ostree_repo_prune_static_deltas (repo, checksum, cancellable, error))
-                goto out;
-            }
+          /* It's newer, traverse it */
+          if (!ostree_repo_traverse_commit_union (repo, next_checksum, 0, reachable,
+                                                  cancellable, error))
+            return FALSE;
+
+          g_free (next_checksum);
+          next_checksum = ostree_commit_get_parent (commit);
+          if (next_checksum)
+            g_clear_pointer (&commit, (GDestroyNotify)g_variant_unref);
           else
-            {
-              if (!ostree_repo_delete_object (repo, OSTREE_OBJECT_TYPE_COMMIT, checksum, cancellable, error))
-                goto out;
-            }
+            break; /* No parent, we're done */
         }
+      else
+        break; /* It's older than our spec, we're done */
     }
 
-  ret = TRUE;
-
- out:
-  return ret;
+  return TRUE;
 }
 
 gboolean
@@ -185,6 +153,9 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
   if (!opt_no_prune && !ostree_ensure_repo_writable (repo, error))
     goto out;
 
+  /* Special handling for explicit commit deletion here - we do this
+   * first.
+   */
   if (opt_delete_commit)
     {
       if (opt_no_prune)
@@ -200,26 +171,133 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError *
         else if (!delete_commit (repo, opt_delete_commit, cancellable, error))
           goto out;
     }
-  if (opt_keep_younger_than)
-    {
-      if (opt_no_prune)
-        {
-          ot_util_usage_error (context, "Cannot specify both --keep-younger-than and --no-prune", error);
-          goto out;
-        }
-      if (!prune_commits_keep_younger_than_date (repo, opt_keep_younger_than, cancellable, error))
-        goto out;
-    }
 
   if (opt_refs_only)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
   if (opt_no_prune)
     pruneflags |= OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE;
 
-  if (!ostree_repo_prune (repo, pruneflags, opt_depth,
-                          &n_objects_total, &n_objects_pruned, &objsize_total,
-                          cancellable, error))
-    goto out;
+  /* If no newer more complex options are specified, drop down to the original
+   * prune API - both to avoid code duplication, and to keep it run from the
+   * test suite.
+   */
+  if (!(opt_retain_branch_depth || opt_keep_younger_than))
+    {
+      if (!ostree_repo_prune (repo, pruneflags, opt_depth,
+                              &n_objects_total, &n_objects_pruned, &objsize_total,
+                              cancellable, error))
+        goto out;
+    }
+  else
+    {
+      g_autoptr(GHashTable) all_refs = NULL;
+      g_autoptr(GHashTable) reachable = ostree_repo_traverse_new_reachable ();
+      g_autoptr(GHashTable) retain_branch_depth = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+      struct timespec keep_younger_than_ts;
+      GHashTableIter hash_iter;
+      gpointer key, value;
+
+      /* Otherwise, the default is --refs-only; we set this just as a note */
+      opt_refs_only = TRUE;
+
+      if (opt_keep_younger_than)
+        {
+          if (!parse_datetime (&keep_younger_than_ts, opt_keep_younger_than, NULL))
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Could not parse '%s'", opt_keep_younger_than);
+              goto out;
+            }
+        }
+
+      for (char **iter = opt_retain_branch_depth; iter && *iter; iter++)
+        {
+          /* bd should look like BRANCH=DEPTH where DEPTH is an int */
+          const char *bd = *iter;
+          const char *eq = strchr (bd, '=');
+          const char *depthstr;
+          gint64 depth;
+          char *endptr;
+
+          if (!eq)
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Invalid value %s, must specify BRANCH=DEPTH",
+                           bd);
+              goto out;
+            }
+          depthstr = eq + 1;
+          errno = EPERM;
+          depth = g_ascii_strtoll (depthstr, &endptr, 10);
+          if (depth == 0)
+            {
+              if (errno == EINVAL)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Out of range depth %s", depthstr);
+                  goto out;
+                }
+              else if (endptr == depthstr)
+                {
+                  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                               "Invalid depth %s", depthstr);
+                  goto out;
+                }
+            }
+          g_hash_table_insert (retain_branch_depth, g_strndup (bd, eq - bd),
+                               GINT_TO_POINTER ((int)depth));
+        }
+
+      /* We start from the refs */
+      if (!ostree_repo_list_refs (repo, NULL, &all_refs,
+                                  cancellable, error))
+        return FALSE;
+
+      g_hash_table_iter_init (&hash_iter, all_refs);
+      while (g_hash_table_iter_next (&hash_iter, &key, &value))
+        {
+          const char *checksum = value;
+          gpointer depthp = g_hash_table_lookup (retain_branch_depth, key);
+          gint depth;
+
+          /* Here, we handle a spec like
+           * --retain-branch-depth=myos/x86_64/stable=-1
+           * --retain-branch-depth=myos/x86_64/dev=5
+           */
+          if (depthp)
+            depth = GPOINTER_TO_INT(depthp);
+          else if (opt_keep_younger_than)
+            {
+              if (!traverse_keep_younger_than (repo, checksum,
+                                               &keep_younger_than_ts,
+                                               reachable,
+                                               cancellable, error))
+                goto out;
+
+              /* Okay, we handled the younger-than case; the other
+               * two fall through to plain depth-based handling below.
+               */
+              continue;  /* Note again, we're skipping the below bit */
+            }
+          else
+            depth = opt_depth; /* No --retain-branch-depth for this branch, use
+                                  the global default */
+
+          g_debug ("Finding objects to keep for commit %s", checksum);
+          if (!ostree_repo_traverse_commit_union (repo, checksum, depth, reachable,
+                                                  cancellable, error))
+            return FALSE;
+        }
+
+      { OstreeRepoPruneOptions opts = { pruneflags, reachable };
+        if (!ostree_repo_prune_from_reachable (repo, &opts,
+                                               &n_objects_total,
+                                               &n_objects_pruned,
+                                               &objsize_total,
+                                               cancellable, error))
+          goto out;
+      }
+    }
 
   formatted_freed_size = g_format_size_full (objsize_total, 0);
 
diff --git a/tests/test-prune.sh b/tests/test-prune.sh
index ce14ce67..51ec7948 100755
--- a/tests/test-prune.sh
+++ b/tests/test-prune.sh
@@ -25,7 +25,7 @@ skip_without_user_xattrs
 
 setup_fake_remote_repo1 "archive-z2"
 
-echo '1..3'
+echo '1..5'
 
 cd ${test_tmpdir}
 mkdir repo
@@ -49,6 +49,7 @@ find repo | grep \.commit$ | wc -l > commitcount
 assert_file_has_content commitcount "^3$"
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_file_has_content tombstonecommitcount "^0$"
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=1 -v
 find repo | grep \.commit$ | wc -l > commitcount
@@ -73,6 +74,7 @@ find repo/objects -name '*.commit' | wc -l > commitcount
 assert_file_has_content commitcount "^1$"
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_not_file_has_content tombstonecommitcount "^0$"
+$OSTREE fsck
 
 # and that tombstone are deleted once the commits are pulled again
 ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test
@@ -83,6 +85,7 @@ COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo log test | grep ^commit | cu
 ${CMD_PREFIX} ostree --repo=repo prune --delete-commit=$COMMIT_TO_DELETE
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_file_has_content tombstonecommitcount "^1$"
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v
 find repo/objects -name '*.commit' | wc -l > commitcount
@@ -94,6 +97,7 @@ assert_file_has_content commitcount "^3$"
 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="2015-10-29 12:43:29 +0000"
 find repo/objects -name '*.commit' | wc -l > commitcount
 assert_file_has_content commitcount "^2$"
+$OSTREE fsck
 
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v
@@ -112,6 +116,7 @@ oldcommit_rev=$($OSTREE --repo=repo rev-parse oldcommit)
 $OSTREE ls ${oldcommit_rev}
 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago"
 $OSTREE ls ${oldcommit_rev}
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test
 ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="November 05 1955"
@@ -124,6 +129,7 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount
 assert_file_has_content deltascount "^2$"
 COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo rev-parse test)
 ${CMD_PREFIX} ostree --repo=repo prune --static-deltas-only --delete-commit=$COMMIT_TO_DELETE
+${CMD_PREFIX} ostree --repo=repo fsck
 ${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount
 assert_file_has_content deltascount "^1$"
 ${CMD_PREFIX} ostree --repo=repo static-delta generate test
@@ -178,3 +184,62 @@ ${CMD_PREFIX} ostree --repo=child-repo prune --refs-only --depth=0
 assert_has_n_objects child-repo 3
 
 echo "ok prune with parent repo"
+
+# Delete all the above since I can't be bothered to think about how new tests
+# would interact. We make a new repo test suite, then clone it
+# for "subtests" below with reinitialize_datesnap_repo()
+rm repo datetest-snapshot-repo -rf
+${CMD_PREFIX} ostree --repo=datetest-snapshot-repo init --mode=archive
+# Some ancient commits on the both a stable/dev branch
+for day in $(seq 5); do
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=stable -m test -s "old stable build $day" tree --timestamp="October $day 1985"
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=dev -m test -s "old dev build $day" tree --timestamp="October $day 1985"
+done
+# And some new ones
+for x in $(seq 3); do
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=stable -m test -s "new stable build $x" tree
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=dev -m test -s "new dev build $x" tree
+done
+find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^16$"
+
+# Snapshot the above
+reinitialize_datesnap_repo() {
+    rm repo -rf
+    ${CMD_PREFIX} ostree --repo=repo init --mode=archive
+    ${CMD_PREFIX} ostree --repo=repo pull-local --depth=-1 datetest-snapshot-repo 
+}
+
+# This test prunes with both younger than as well as a full strong ref to the
+# stable branch
+reinitialize_datesnap_repo
+# First, a quick test of invalid input
+if ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=BACON 2>err.txt; then
+    assert_not_reached "BACON is a number?!"
+fi
+assert_file_has_content err.txt 'Invalid depth BACON'
+${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=-1
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^11$"
+# Double check our backup is unchanged
+find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^16$"
+$OSTREE fsck
+
+# Again but this time only retain 6 (5+1) commits on stable.  This should drop
+# out 8 - 6 = 2 commits (so the 11 above minus 2 = 9)
+${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=5
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^9$"
+$OSTREE fsck
+echo "ok retain branch depth and keep-younger-than"
+
+# Just stable branch ref, we should prune everything except the tip of dev,
+# so 8 stable + 1 dev = 9
+reinitialize_datesnap_repo
+${CMD_PREFIX} ostree --repo=repo prune --depth=0 --retain-branch-depth=stable=-1
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^9$"
+$OSTREE fsck
+
+echo "ok retain branch depth (alone)"

From d1f4d9472005ab3d3614c56cc06b1c04f8c1c995 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 18 Jan 2017 13:35:42 -0500
Subject: [PATCH 26/27] tests: Add a big (many objects) pull

This would be more likely to tickle things like
https://github.com/ostreedev/ostree/issues/601
reliably.

Also, while working on the curl backend, I hit on the fact that curl doesn't
queue (by default, you can enable) and will happily create 20000+ concurrent TCP
connections if you try. Having this test would have made that more likely to
fail.

Closes: #650
Approved by: giuseppe
---
 Makefile-tests.am       |   1 +
 tests/test-pull-many.sh | 100 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100755 tests/test-pull-many.sh

diff --git a/Makefile-tests.am b/Makefile-tests.am
index 957b15a8..a0c05488 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -66,6 +66,7 @@ dist_test_scripts = \
 	tests/test-pull-resume.sh \
 	tests/test-pull-repeated.sh \
 	tests/test-pull-untrusted.sh \
+	tests/test-pull-many.sh \
 	tests/test-pull-override-url.sh \
 	tests/test-local-pull.sh \
 	tests/test-local-pull-depth.sh \
diff --git a/tests/test-pull-many.sh b/tests/test-pull-many.sh
new file mode 100755
index 00000000..d90280d5
--- /dev/null
+++ b/tests/test-pull-many.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+cd ${test_tmpdir}
+rm ostree-srv/gnomerepo/ -rf
+mkdir ostree-srv/gnomerepo/
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo init --mode=archive
+
+echo '1..1'
+
+# Simulate a large archive pull from scratch.  Large here is
+# something like the Fedora Atomic Workstation branch which has
+# objects: meta: 7497 content: 103541
+# 9443 directories, 7097 symlinks, 112832 regfiles
+# So we'll make ~11 files per dir, with one of them a symlink
+
+cd ${test_tmpdir}
+rm main -rf
+mkdir main
+cd main
+ndirs=9443
+depth=0
+echo "$(date): Generating content..."
+set +x  # No need to spam the logs for this
+while [ $ndirs -gt 0 ]; do
+    # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up
+    x=$(($ndirs % 3))
+    case $x in
+        0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;;
+        1|2) if [ $depth -lt 9 ]; then
+                 mkdir dir-${ndirs}
+                 cd dir-${ndirs}
+                 depth=$((depth+1))
+             else
+                 if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi
+             fi ;;
+    esac
+
+    # One symlink - we use somewhat predictable content to have dupes
+    ln -s $(($x % 20)) link-$ndirs
+    # 10 files
+    nfiles=10
+    while [ $nfiles -gt 0 ]; do
+        echo file-$ndirs-$nfiles > f$ndirs-$nfiles
+        nfiles=$((nfiles-1))
+    done
+    ndirs=$((ndirs-1))
+done
+set -x
+cd ${test_tmpdir}
+mkdir build-repo
+${CMD_PREFIX} ostree --repo=build-repo init --mode=bare-user
+echo "$(date): Committing content..."
+${CMD_PREFIX} ostree --repo=build-repo commit -b main -s 'big!' --tree=dir=main
+for x in commit dirtree dirmeta file; do
+    find build-repo/objects -name '*.'${x} |wc -l > ${x}count
+    echo "$x: " $(cat ${x}count)
+done
+assert_file_has_content commitcount '^1$'
+assert_file_has_content dirmetacount '^1$'
+assert_file_has_content filecount '^94433$'
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo pull-local build-repo
+
+echo "$(date): Pulling content..."
+rm repo -rf
+${CMD_PREFIX} ostree --repo=repo init --mode=archive
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+${CMD_PREFIX} ostree --repo=repo pull --mirror origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+for x in commit dirtree dirmeta filez; do
+    find repo/objects -name '*.'${x} |wc -l > ${x}count
+    echo "$x: " $(cat ${x}count)
+done
+assert_file_has_content commitcount '^1$'
+assert_file_has_content dirmetacount '^1$'
+assert_file_has_content filezcount '^94433$'
+
+echo "ok"

From 6517a8a27a1386e7cb5482e7cb2919fe92721ccf Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Mon, 23 Jan 2017 15:22:30 -0500
Subject: [PATCH 27/27] Release 2017.1

Just bugfixes, but it's time to ship them.

Closes: #653
Approved by: jlebon
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index c59b3f7b..e3f2d12a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,6 @@
 AC_PREREQ([2.63])
 dnl If incrementing the version here, remember to update libostree.sym too
-AC_INIT([ostree], [2016.15], [walters@verbum.org])
+AC_INIT([ostree], [2017.1], [walters@verbum.org])
 AC_CONFIG_HEADER([config.h])
 AC_CONFIG_MACRO_DIR([buildutil])
 AC_CONFIG_AUX_DIR([build-aux])