New upstream version 2017.15

src/libostree/libostree-devel.sym

@@ -18,9 +18,8 @@
 ***/
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
-
-LIBOSTREE_2017.15 {
-} LIBOSTREE_2017.14;
+LIBOSTREE_2017.16 {
+} LIBOSTREE_2017.15;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste

src/libostree/libostree-experimental.sym

@@ -94,4 +94,8 @@ LIBOSTREE_2017.14_EXPERIMENTAL {
 global:
   ostree_remote_get_type;
   ostree_remote_get_url;
+  ostree_repo_auto_lock_cleanup;
+  ostree_repo_auto_lock_push;
+  ostree_repo_lock_pop;
+  ostree_repo_lock_push;
 } LIBOSTREE_2017.13_EXPERIMENTAL;

src/libostree/libostree-released.sym

@@ -445,6 +445,12 @@ global:
 LIBOSTREE_2017.14 {
 } LIBOSTREE_2017.13;
 
+LIBOSTREE_2017.15 {
+  ostree_repo_fsck_object;
+  ostree_repo_mark_commit_partial;
+  ostree_break_hardlink;
+} LIBOSTREE_2017.14;
+
 /* NOTE: Only add more content here in release commits!  See the
  * comments at the top of this file.
  */

src/libostree/ostree-autocleanups.h

@@ -59,6 +59,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref)
 G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear)
 
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free)
 G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref)

src/libostree/ostree-cmdprivate.c

@@ -22,6 +22,7 @@
 #include "ostree-cmdprivate.h"
 #include "ostree-repo-private.h"
 #include "ostree-core-private.h"
+#include "ostree-repo-pull-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-sysroot.h"
 #include "ostree-bootloader-grub2.h"
@@ -48,7 +49,8 @@ ostree_cmd__private__ (void)
     impl_ostree_generate_grub2_config,
     _ostree_repo_static_delta_dump,
     _ostree_repo_static_delta_query_exists,
-    _ostree_repo_static_delta_delete
+    _ostree_repo_static_delta_delete,
+    _ostree_repo_verify_bindings
   };
 
   return &table;

src/libostree/ostree-cmdprivate.h

@@ -31,6 +31,7 @@ typedef struct {
   gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
   gboolean (* ostree_static_delta_query_exists) (OstreeRepo *repo, const char *delta_id, gboolean *out_exists, GCancellable *cancellable, GError **error);
   gboolean (* ostree_static_delta_delete) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
+  gboolean (* ostree_repo_verify_bindings) (const char *collection_id, const char *ref_name, GVariant *commit, GError **error);
 } OstreeCmdPrivateVTable;
 
 /* Note this not really "public", we just export the symbol, but not the header */

src/libostree/ostree-core.c

@@ -749,6 +749,107 @@ ostree_content_file_parse (gboolean                compressed,
                                        cancellable, error);
 }
 
+static gboolean
+break_symhardlink (int                dfd,
+                   const char        *path,
+                   struct stat       *stbuf,
+                   GLnxFileCopyFlags  copyflags,
+                   GCancellable      *cancellable,
+                   GError           **error)
+{
+  guint count;
+  gboolean copy_success = FALSE;
+  char *path_tmp = glnx_strjoina (path, ".XXXXXX");
+
+  for (count = 0; count < 100; count++)
+    {
+      g_autoptr(GError) tmp_error = NULL;
+
+      glnx_gen_temp_name (path_tmp);
+
+      if (!glnx_file_copy_at (dfd, path, stbuf, dfd, path_tmp, copyflags,
+                              cancellable, &tmp_error))
+        {
+          if (g_error_matches (tmp_error, G_IO_ERROR, G_IO_ERROR_EXISTS))
+            continue;
+          g_propagate_error (error, g_steal_pointer (&tmp_error));
+          return FALSE;
+        }
+
+      copy_success = TRUE;
+      break;
+    }
+
+  if (!copy_success)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_EXISTS,
+                   "Exceeded limit of %u file creation attempts", count);
+      return FALSE;
+    }
+
+  if (!glnx_renameat (dfd, path_tmp, dfd, path, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+/**
+ * ostree_break_hardlink:
+ * @dfd: Directory fd
+ * @path: Path relative to @dfd
+ * @skip_xattrs: Do not copy extended attributes
+ * @error: error
+ *
+ * In many cases using libostree, a program may need to "break"
+ * hardlinks by performing a copy.  For example, in order to
+ * logically append to a file.
+ *
+ * This function performs full copying, including e.g. extended
+ * attributes and permissions of both regular files and symbolic links.
+ *
+ * If the file is not hardlinked, this function does nothing and
+ * returns successfully.
+ *
+ * This function does not perform synchronization via `fsync()` or
+ * `fdatasync()`; the idea is this will commonly be done as part
+ * of an `ostree_repo_commit_transaction()`, which itself takes
+ * care of synchronization.
+ *
+ * Since: 2017.15
+ */
+gboolean ostree_break_hardlink (int               dfd,
+                                const char       *path,
+                                gboolean          skip_xattrs,
+                                GCancellable     *cancellable,
+                                GError          **error)
+{
+  struct stat stbuf;
+
+  if (!glnx_fstatat (dfd, path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+
+  if (stbuf.st_nlink <= 1)
+    return TRUE;  /* Note early return */
+
+  const GLnxFileCopyFlags copyflags = skip_xattrs ? GLNX_FILE_COPY_NOXATTRS : 0;
+
+  if (S_ISREG (stbuf.st_mode))
+    /* Note it's now completely safe to copy a file to itself,
+     * as glnx_file_copy_at() is documented to do an O_TMPFILE + rename()
+     * with GLNX_FILE_COPY_OVERWRITE.
+     */
+    return glnx_file_copy_at (dfd, path, &stbuf, dfd, path,
+                              copyflags | GLNX_FILE_COPY_OVERWRITE,
+                              cancellable, error);
+  else if (S_ISLNK (stbuf.st_mode))
+    return break_symhardlink (dfd, path, &stbuf, copyflags,
+                              cancellable, error);
+  else
+    return glnx_throw (error, "Unsupported type for entry '%s'", path);
+
+  return TRUE;
+}
+
 /**
  * ostree_checksum_file_from_input:
  * @file_info: File information

src/libostree/ostree-core.h

@@ -438,6 +438,13 @@ gboolean ostree_checksum_file (GFile             *f,
                                GCancellable      *cancellable,
                                GError           **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_break_hardlink (int               dfd,
+                                const char       *path,
+                                gboolean          skip_xattrs,
+                                GCancellable     *cancellable,
+                                GError          **error);
+
 /**
  * OstreeChecksumFlags:
  *

src/libostree/ostree-fetcher-curl.c

@@ -771,13 +771,18 @@ initiate_next_curl_request (FetcherRequest *req,
    * there are numerous HTTP/2 fixes since the original version in
    * libcurl 7.43.0.
    */
+#ifdef BUILDOPT_HTTP2
+  if (!(self->config_flags & OSTREE_FETCHER_FLAGS_DISABLE_HTTP2))
+    {
 #if CURL_AT_LEAST_VERSION(7, 51, 0)
-  curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+      curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 #endif
-  /* https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c */
+      /* https://github.com/curl/curl/blob/curl-7_53_0/docs/examples/http2-download.c */
 #if (CURLPIPE_MULTIPLEX > 0)
-  /* wait for pipe connection to confirm */
-  curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L);
+      /* wait for pipe connection to confirm */
+      curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L);
+#endif
+    }
 #endif
   curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
   if (g_getenv ("OSTREE_DEBUG_HTTP"))

src/libostree/ostree-fetcher.h

@@ -50,7 +50,8 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcher, g_object_unref)
 typedef enum {
   OSTREE_FETCHER_FLAGS_NONE = 0,
   OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0),
-  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1)
+  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1),
+  OSTREE_FETCHER_FLAGS_DISABLE_HTTP2 = (1 << 2),
 } OstreeFetcherConfigFlags;
 
 typedef enum {

src/libostree/ostree-repo-file.c

@@ -508,7 +508,7 @@ ostree_repo_file_get_parent (GFile *file)
 {
   OstreeRepoFile *self = OSTREE_REPO_FILE (file);
 
-  return g_object_ref (self->parent);
+  return (GFile*)g_object_ref (self->parent);
 }
 
 static GFile *
@@ -621,7 +621,7 @@ ostree_repo_file_resolve_relative_path (GFile      *file,
       g_assert (*relative_path == '/');
 
       if (strcmp (relative_path, "/") == 0)
-        return g_object_ref (ostree_repo_file_get_root (self)); 
+        return (GFile*)g_object_ref (ostree_repo_file_get_root (self));
 
       if (self->parent)
         return ostree_repo_file_resolve_relative_path ((GFile*)ostree_repo_file_get_root (self),

src/libostree/ostree-repo-private.h

@@ -82,6 +82,15 @@ typedef enum {
   OSTREE_REPO_SYSROOT_KIND_IS_SYSROOT_OSTREE, /* We match /ostree/repo */
 } OstreeRepoSysrootKind;
 
+typedef struct {
+  GHashTable *refs;  /* (element-type utf8 utf8) */
+  GHashTable *collection_refs;  /* (element-type OstreeCollectionRef utf8) */
+  OstreeRepoTransactionStats stats;
+  /* Implementation of min-free-space-percent */
+  gulong blocksize;
+  fsblkcnt_t max_blocks;
+} OstreeRepoTxn;
+
 /**
  * OstreeRepo:
  *
@@ -109,13 +118,9 @@ struct OstreeRepo {
   GWeakRef sysroot; /* Weak to avoid a circular ref; see also `is_system` */
   char *remotes_config_dir;
 
-  GHashTable *txn_refs;  /* (element-type utf8 utf8) */
-  GHashTable *txn_collection_refs;  /* (element-type OstreeCollectionRef utf8) */
-  GMutex txn_stats_lock;
-  OstreeRepoTransactionStats txn_stats;
-  /* Implementation of min-free-space-percent */
-  gulong txn_blocksize;
-  fsblkcnt_t max_txn_blocks;
+  GMutex txn_lock;
+  OstreeRepoTxn txn;
+  gboolean txn_locked;
 
   GMutex cache_lock;
   guint dirmeta_cache_refcount;
@@ -153,6 +158,7 @@ struct OstreeRepo {
   guint64 tmp_expiry_seconds;
   gchar *collection_id;
   gboolean add_remotes_config_dir; /* Add new remotes in remotes.d dir */
+  gint lock_timeout_seconds;
 
   OstreeRepo *parent_repo;
 };
@@ -432,6 +438,32 @@ _ostree_repo_get_remote_inherited (OstreeRepo  *self,
 
 #ifndef OSTREE_ENABLE_EXPERIMENTAL_API
 
+/* All the locking APIs below are duplicated in ostree-repo.h. Remove the ones
+ * here once it's no longer experimental.
+ */
+
+typedef enum {
+  OSTREE_REPO_LOCK_SHARED,
+  OSTREE_REPO_LOCK_EXCLUSIVE
+} OstreeRepoLockType;
+
+gboolean      ostree_repo_lock_push (OstreeRepo          *self,
+                                     OstreeRepoLockType   lock_type,
+                                     GCancellable        *cancellable,
+                                     GError             **error);
+gboolean      ostree_repo_lock_pop (OstreeRepo    *self,
+                                    GCancellable  *cancellable,
+                                    GError       **error);
+
+typedef OstreeRepo OstreeRepoAutoLock;
+
+OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo          *self,
+                                                 OstreeRepoLockType   lock_type,
+                                                 GCancellable        *cancellable,
+                                                 GError             **error);
+void          ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup)
+
 const gchar * ostree_repo_get_collection_id (OstreeRepo   *self);
 gboolean      ostree_repo_set_collection_id (OstreeRepo   *self,
                                              const gchar  *collection_id,

src/libostree/ostree-repo-prune.c

@@ -23,6 +23,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-autocleanups.h"
 #include "otutil.h"
 
 typedef struct {
@@ -35,16 +36,6 @@ typedef struct {
   guint64 freed_bytes;
 } OtPruneData;
 
-static gboolean
-prune_commitpartial_file (OstreeRepo    *repo,
-                          const char    *checksum,
-                          GCancellable  *cancellable,
-                          GError       **error)
-{
-  g_autofree char *path = _ostree_get_commitpartial_path (checksum);
-  return ot_ensure_unlinked_at (repo->repo_dir_fd, path, error);
-}
-
 static gboolean
 maybe_prune_loose_object (OtPruneData        *data,
                           OstreeRepoPruneFlags    flags,
@@ -67,7 +58,7 @@ maybe_prune_loose_object (OtPruneData        *data,
 
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
             {
-              if (!prune_commitpartial_file (data->repo, checksum, cancellable, error))
+              if (!ostree_repo_mark_commit_partial (data->repo, checksum, FALSE, error))
                 return FALSE;
             }
 
@@ -160,12 +151,20 @@ _ostree_repo_prune_tmp (OstreeRepo *self,
  * Prune static deltas, if COMMIT is specified then delete static delta files only
  * targeting that commit; otherwise any static delta of non existing commits are
  * deleted.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit,
                                  GCancellable      *cancellable,
                                  GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GPtrArray) deltas = NULL;
   if (!ostree_repo_list_static_delta_names (self, &deltas,
                                             cancellable, error))
@@ -286,6 +285,8 @@ repo_prune_internal (OstreeRepo        *self,
  * Use the %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE to just determine
  * statistics on objects that would be deleted, without actually
  * deleting them.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune (OstreeRepo        *self,
@@ -297,6 +298,12 @@ ostree_repo_prune (OstreeRepo        *self,
                    GCancellable      *cancellable,
                    GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GHashTable) objects = NULL;
   gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
 
@@ -391,6 +398,8 @@ ostree_repo_prune (OstreeRepo        *self,
  *
  * The %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE flag may be specified to just determine
  * statistics on objects that would be deleted, without actually deleting them.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune_from_reachable (OstreeRepo        *self,
@@ -401,6 +410,12 @@ ostree_repo_prune_from_reachable (OstreeRepo        *self,
                                   GCancellable      *cancellable,
                                   GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GHashTable) objects = NULL;
 
   if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,

src/libostree/ostree-repo-pull-private.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright © 2017 Endless Mobile, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#include "ostree-core.h"
+
+G_BEGIN_DECLS
+
+gboolean
+_ostree_repo_verify_bindings (const char  *collection_id,
+                              const char  *ref_name,
+                              GVariant    *commit,
+                              GError     **error);
+
+G_END_DECLS

src/libostree/ostree-repo-pull.c

@@ -32,6 +32,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-repo-pull-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
 #include "ostree-fetcher-util.h"
@@ -557,21 +558,6 @@ fetch_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
                                                 cancellable, error);
 }
 
-static gboolean
-write_commitpartial_for (OtPullData *pull_data,
-                         const char *checksum,
-                         GError **error)
-{
-  g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
-  glnx_autofd int fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644);
-  if (fd == -1)
-    {
-      if (errno != EEXIST)
-        return glnx_throw_errno_prefix (error, "open(%s)", commitpartial_path);
-    }
-  return TRUE;
-}
-
 static void
 enqueue_one_object_request (OtPullData                *pull_data,
                             const char                *checksum,
@@ -1266,7 +1252,7 @@ meta_fetch_on_complete (GObject           *object,
                                             pull_data->cancellable, error))
             goto out;
 
-          if (!write_commitpartial_for (pull_data, checksum, error))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
             goto out;
         }
 
@@ -1475,30 +1461,40 @@ get_remote_repo_collection_id (OtPullData *pull_data)
 }
 #endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
 
-/* Verify the ref and collection bindings.
+#endif  /* HAVE_LIBCURL_OR_LIBSOUP */
+
+/**
+ * _ostree_repo_verify_bindings:
+ * @collection_id: (nullable): Locally specified collection ID for the remote
+ *    the @commit was retrieved from, or %NULL if none is configured
+ * @ref_name: (nullable): Ref name the commit was retrieved using, or %NULL if
+ *    the commit was retrieved by checksum
+ * @commit: Commit data to check
+ * @error: Return location for a #GError, or %NULL
+ *
+ * Verify the ref and collection bindings.
  *
  * The ref binding is verified only if it exists. But if we have the
- * collection ID specified in the remote configuration then the ref
- * binding must exist, otherwise the verification will fail. Parts of
- * the verification can be skipped by passing NULL to the requested_ref
- * parameter (in case we requested a checksum directly, without looking it up
- * from a ref).
+ * collection ID specified in the remote configuration (@collection_id is
+ * non-%NULL) then the ref binding must exist, otherwise the verification will
+ * fail. Parts of the verification can be skipped by passing %NULL to the
+ * @ref_name parameter (in case we requested a checksum directly, without
+ * looking it up from a ref).
  *
  * The collection binding is verified only when we have collection ID
  * specified in the remote configuration. If it is specified, then the
  * binding must exist and must be equal to the remote repository
  * collection ID.
+ *
+ * Returns: %TRUE if bindings are correct, %FALSE otherwise
+ * Since: 2017.14
  */
-static gboolean
-verify_bindings (OtPullData                 *pull_data,
-                 GVariant                   *commit,
-                 const OstreeCollectionRef  *requested_ref,
-                 GError                    **error)
+gboolean
+_ostree_repo_verify_bindings (const char  *collection_id,
+                              const char  *ref_name,
+                              GVariant    *commit,
+                              GError     **error)
 {
-  g_autofree char *remote_collection_id = NULL;
-#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
-  remote_collection_id = get_remote_repo_collection_id (pull_data);
-#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
   g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0);
   g_autofree const char **refs = NULL;
   if (!g_variant_lookup (metadata,
@@ -1510,17 +1506,17 @@ verify_bindings (OtPullData                 *pull_data,
        * we certainly will not verify the collection binding in the
        * commit.
        */
-      if (remote_collection_id == NULL)
+      if (collection_id == NULL)
         return TRUE;
 
       return glnx_throw (error,
-                         "expected commit metadata to have ref "
+                         "Expected commit metadata to have ref "
                          "binding information, found none");
     }
 
-  if (requested_ref != NULL)
+  if (ref_name != NULL)
     {
-      if (!g_strv_contains ((const char *const *) refs, requested_ref->ref_name))
+      if (!g_strv_contains ((const char *const *) refs, ref_name))
         {
           g_autoptr(GString) refs_dump = g_string_new (NULL);
           const char *refs_str;
@@ -1543,35 +1539,37 @@ verify_bindings (OtPullData                 *pull_data,
               refs_str = "no refs";
             }
 
-          return glnx_throw (error, "commit has no requested ref ‘%s’ "
+          return glnx_throw (error, "Commit has no requested ref ‘%s’ "
                              "in ref binding metadata (%s)",
-                             requested_ref->ref_name, refs_str);
+                             ref_name, refs_str);
         }
     }
 
-  if (remote_collection_id != NULL)
+  if (collection_id != NULL)
     {
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
-      const char *collection_id;
+      const char *collection_id_binding;
       if (!g_variant_lookup (metadata,
                              OSTREE_COMMIT_META_KEY_COLLECTION_BINDING,
                              "&s",
-                             &collection_id))
+                             &collection_id_binding))
         return glnx_throw (error,
-                           "expected commit metadata to have collection ID "
+                           "Expected commit metadata to have collection ID "
                            "binding information, found none");
-      if (!g_str_equal (collection_id, remote_collection_id))
+      if (!g_str_equal (collection_id_binding, collection_id))
         return glnx_throw (error,
-                           "commit has collection ID ‘%s’ in collection binding "
+                           "Commit has collection ID ‘%s’ in collection binding "
                            "metadata, while the remote it came from has "
                            "collection ID ‘%s’",
-                           collection_id, remote_collection_id);
+                           collection_id_binding, collection_id);
 #endif
     }
 
   return TRUE;
 }
 
+#ifdef HAVE_LIBCURL_OR_LIBSOUP
+
 /* Look at a commit object, and determine whether there are
  * more things to fetch.
  */
@@ -1626,7 +1624,13 @@ scan_commit_object (OtPullData                 *pull_data,
   /* If ref is non-NULL then the commit we fetched was requested through the
    * branch, otherwise we requested a commit checksum without specifying a branch.
    */
-  if (!verify_bindings (pull_data, commit, ref, error))
+  g_autofree char *remote_collection_id = NULL;
+#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+  remote_collection_id = get_remote_repo_collection_id (pull_data);
+#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
+  if (!_ostree_repo_verify_bindings (remote_collection_id,
+                                     (ref != NULL) ? ref->ref_name : NULL,
+                                     commit, error))
     return glnx_prefix_error (error, "Commit %s", checksum);
 
   if (pull_data->timestamp_check)
@@ -1802,7 +1806,7 @@ scan_one_metadata_object (OtPullData                 *pull_data,
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
           /* mark as partial to ensure we scan the commit below */
-          if (!write_commitpartial_for (pull_data, checksum, error))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
             return FALSE;
         }
 
@@ -1835,7 +1839,7 @@ scan_one_metadata_object (OtPullData                 *pull_data,
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
             {
               /* mark as partial to ensure we scan the commit below */
-              if (!write_commitpartial_for (pull_data, checksum, error))
+              if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
                 return FALSE;
             }
           if (!_ostree_repo_import_object (pull_data->repo, refd_repo,
@@ -2690,6 +2694,15 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (gzip)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
 
+  { gboolean http2 = TRUE;
+    if (!ostree_repo_get_remote_boolean_option (self, remote_name,
+                                                "http2", TRUE,
+                                                &http2, error))
+      goto out;
+    if (!http2)
+      fetcher_flags |= OSTREE_FETCHER_FLAGS_DISABLE_HTTP2;
+  }
+
   fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
 
   {
@@ -4312,15 +4325,13 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     {
       GLNX_HASH_TABLE_FOREACH_V (requested_refs_to_fetch, const char*, checksum)
         {
-          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
-          if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, FALSE, error))
             goto out;
         }
 
       GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit)
         {
-          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit);
-          if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error))
             goto out;
         }
     }

src/libostree/ostree-repo-refs.c

@@ -726,13 +726,17 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
  * @self: Repo
  * @refspec_prefix: (allow-none): Only list refs which match this prefix
  * @out_all_refs: (out) (element-type utf8 utf8) (transfer container):
- *    Mapping from ref to checksum
+ *    Mapping from refspec to checksum
  * @cancellable: Cancellable
  * @error: Error
  *
  * If @refspec_prefix is %NULL, list all local and remote refspecs,
  * with their current values in @out_all_refs.  Otherwise, only list
  * refspecs which have @refspec_prefix as a prefix.
+ *
+ * @out_all_refs will be returned as a mapping from refspecs (including the
+ * remote name) to checksums. If @refspec_prefix is non-%NULL, it will be
+ * removed as a prefix from the hash table keys.
  */
 gboolean
 ostree_repo_list_refs (OstreeRepo       *self,
@@ -752,16 +756,18 @@ ostree_repo_list_refs (OstreeRepo       *self,
  * @self: Repo
  * @refspec_prefix: (allow-none): Only list refs which match this prefix
  * @out_all_refs: (out) (element-type utf8 utf8) (transfer container):
- *    Mapping from ref to checksum
+ *    Mapping from refspec to checksum
  * @flags: Options controlling listing behavior
  * @cancellable: Cancellable
  * @error: Error
  *
  * If @refspec_prefix is %NULL, list all local and remote refspecs,
  * with their current values in @out_all_refs.  Otherwise, only list
- * refspecs which have @refspec_prefix as a prefix.  Differently from
- * ostree_repo_list_refs(), the prefix will not be removed from the ref
- * name.
+ * refspecs which have @refspec_prefix as a prefix.
+ *
+ * @out_all_refs will be returned as a mapping from refspecs (including the
+ * remote name) to checksums. Differently from ostree_repo_list_refs(), the
+ * @refspec_prefix will not be removed from the refspecs in the hash table.
  */
 gboolean
 ostree_repo_list_refs_ext (OstreeRepo                 *self,

src/libostree/ostree-repo-static-delta-private.h

@@ -128,11 +128,6 @@ _ostree_static_delta_part_open (GInputStream   *part_in,
                                 GCancellable *cancellable,
                                 GError      **error);
 
-gboolean _ostree_static_delta_dump (OstreeRepo     *repo,
-                                    const char *delta_id,
-                                    GCancellable   *cancellable,
-                                    GError        **error);
-
 typedef struct {
   guint n_ops_executed[OSTREE_STATIC_DELTA_N_OPS];
 } OstreeDeltaExecuteStats;

src/libostree/ostree-repo.c

@@ -46,6 +46,9 @@
 #include <sys/file.h>
 #include <sys/statvfs.h>
 
+#define REPO_LOCK_DISABLED (-2)
+#define REPO_LOCK_BLOCKING (-1)
+
 /* ABI Size checks for ostree-repo.h, only for LP64 systems;
  * https://en.wikipedia.org/wiki/64-bit_computing#64-bit_data_models
  *
@@ -156,6 +159,526 @@ G_DEFINE_TYPE (OstreeRepo, ostree_repo, G_TYPE_OBJECT)
 
 #define SYSCONF_REMOTES SHORTENED_SYSCONFDIR "/ostree/remotes.d"
 
+/* Repository locking
+ *
+ * To guard against objects being deleted (e.g., prune) while they're in
+ * use by another operation is accessing them (e.g., commit), the
+ * repository must be locked by concurrent writers.
+ *
+ * The locking is implemented by maintaining a thread local table of
+ * lock stacks per repository. This allows thread safe locking since
+ * each thread maintains its own lock stack. See the OstreeRepoLock type
+ * below.
+ *
+ * The actual locking is done using either open file descriptor locks or
+ * flock locks. This allows the locking to work with concurrent
+ * processes. The lock file is held on the ".lock" file within the
+ * repository.
+ *
+ * The intended usage is to take a shared lock when writing objects or
+ * reading objects in critical sections. Exclusive locks are taken when
+ * deleting objects.
+ *
+ * To allow fine grained locking within libostree, the lock is
+ * maintained as a stack. The core APIs then push or pop from the stack.
+ * When pushing or popping a lock state identical to the existing or
+ * next state, the stack is simply updated. Only when upgrading or
+ * downgrading the lock (changing to/from unlocked, pushing exclusive on
+ * shared or popping exclusive to shared) are actual locking operations
+ * performed.
+ */
+
+static void
+free_repo_lock_table (gpointer data)
+{
+  GHashTable *lock_table = data;
+
+  if (lock_table != NULL)
+    {
+      g_debug ("Free lock table");
+      g_hash_table_destroy (lock_table);
+    }
+}
+
+static GPrivate repo_lock_table = G_PRIVATE_INIT (free_repo_lock_table);
+
+typedef struct {
+  int fd;
+  GQueue stack;
+} OstreeRepoLock;
+
+typedef struct {
+  guint len;
+  int state;
+  const char *name;
+} OstreeRepoLockInfo;
+
+static void
+repo_lock_info (OstreeRepoLock *lock, OstreeRepoLockInfo *out_info)
+{
+  g_assert (lock != NULL);
+  g_assert (out_info != NULL);
+
+  OstreeRepoLockInfo info;
+  info.len = g_queue_get_length (&lock->stack);
+  if (info.len == 0)
+    {
+      info.state = LOCK_UN;
+      info.name = "unlocked";
+    }
+  else
+    {
+      info.state = GPOINTER_TO_INT (g_queue_peek_head (&lock->stack));
+      info.name = (info.state == LOCK_EX) ? "exclusive" : "shared";
+    }
+
+  *out_info = info;
+}
+
+static void
+free_repo_lock (gpointer data)
+{
+  OstreeRepoLock *lock = data;
+
+  if (lock != NULL)
+    {
+      OstreeRepoLockInfo info;
+      repo_lock_info (lock, &info);
+
+      g_debug ("Free lock: state=%s, depth=%u", info.name, info.len);
+      g_queue_clear (&lock->stack);
+      if (lock->fd >= 0)
+        {
+          g_debug ("Closing repo lock file");
+          (void) close (lock->fd);
+        }
+      g_free (lock);
+    }
+}
+
+/* Wrapper to handle flock vs OFD locking based on GLnxLockFile */
+static gboolean
+do_repo_lock (int fd,
+              int flags)
+{
+  int res;
+
+#ifdef F_OFD_SETLK
+  struct flock fl = {
+    .l_type = (flags & ~LOCK_NB) == LOCK_EX ? F_WRLCK : F_RDLCK,
+    .l_whence = SEEK_SET,
+    .l_start = 0,
+    .l_len = 0,
+  };
+
+  res = TEMP_FAILURE_RETRY (fcntl (fd, (flags & LOCK_NB) ? F_OFD_SETLK : F_OFD_SETLKW, &fl));
+#else
+  res = -1;
+  errno = EINVAL;
+#endif
+
+  /* Fallback to flock when OFD locks not available */
+  if (res < 0)
+    {
+      if (errno == EINVAL)
+        res = TEMP_FAILURE_RETRY (flock (fd, flags));
+      if (res < 0)
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+/* Wrapper to handle flock vs OFD unlocking based on GLnxLockFile */
+static gboolean
+do_repo_unlock (int fd,
+                int flags)
+{
+  int res;
+
+#ifdef F_OFD_SETLK
+  struct flock fl = {
+    .l_type = F_UNLCK,
+    .l_whence = SEEK_SET,
+    .l_start = 0,
+    .l_len = 0,
+  };
+
+  res = TEMP_FAILURE_RETRY (fcntl (fd, (flags & LOCK_NB) ? F_OFD_SETLK : F_OFD_SETLKW, &fl));
+#else
+  res = -1;
+  errno = EINVAL;
+#endif
+
+  /* Fallback to flock when OFD locks not available */
+  if (res < 0)
+    {
+      if (errno == EINVAL)
+        res = TEMP_FAILURE_RETRY (flock (fd, LOCK_UN | flags));
+      if (res < 0)
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+push_repo_lock (OstreeRepo          *self,
+                OstreeRepoLockType   lock_type,
+                gboolean             blocking,
+                GError             **error)
+{
+  int flags = (lock_type == OSTREE_REPO_LOCK_EXCLUSIVE) ? LOCK_EX : LOCK_SH;
+  if (!blocking)
+    flags |= LOCK_NB;
+
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  if (lock_table == NULL)
+    {
+      g_debug ("Creating repo lock table");
+      lock_table = g_hash_table_new_full (NULL, NULL, NULL,
+                                          (GDestroyNotify)free_repo_lock);
+      g_private_set (&repo_lock_table, lock_table);
+    }
+
+  OstreeRepoLock *lock = g_hash_table_lookup (lock_table, self);
+  if (lock == NULL)
+    {
+      lock = g_new0 (OstreeRepoLock, 1);
+      g_queue_init (&lock->stack);
+      g_debug ("Opening repo lock file");
+      lock->fd = TEMP_FAILURE_RETRY (openat (self->repo_dir_fd, ".lock",
+                                             O_CREAT | O_RDWR | O_CLOEXEC,
+                                             0600));
+      if (lock->fd < 0)
+        {
+          free_repo_lock (lock);
+          return glnx_throw_errno_prefix (error,
+                                          "Opening lock file %s/.lock failed",
+                                          gs_file_get_path_cached (self->repodir));
+        }
+      g_hash_table_insert (lock_table, self, lock);
+    }
+
+  OstreeRepoLockInfo info;
+  repo_lock_info (lock, &info);
+  g_debug ("Push lock: state=%s, depth=%u", info.name, info.len);
+
+  if (info.state == LOCK_EX)
+    {
+      g_debug ("Repo already locked exclusively, extending stack");
+      g_queue_push_head (&lock->stack, GINT_TO_POINTER (LOCK_EX));
+    }
+  else
+    {
+      int next_state = (flags & LOCK_EX) ? LOCK_EX : LOCK_SH;
+      const char *next_state_name = (flags & LOCK_EX) ? "exclusive" : "shared";
+
+      g_debug ("Locking repo %s", next_state_name);
+      if (!do_repo_lock (lock->fd, flags))
+        return glnx_throw_errno_prefix (error, "Locking repo %s failed",
+                                        next_state_name);
+
+      g_queue_push_head (&lock->stack, GINT_TO_POINTER (next_state));
+    }
+
+  return TRUE;
+}
+
+static gboolean
+pop_repo_lock (OstreeRepo  *self,
+               gboolean     blocking,
+               GError     **error)
+{
+  int flags = blocking ? 0 : LOCK_NB;
+
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  g_return_val_if_fail (lock_table != NULL, FALSE);
+
+  OstreeRepoLock *lock = g_hash_table_lookup (lock_table, self);
+  g_return_val_if_fail (lock != NULL, FALSE);
+  g_return_val_if_fail (lock->fd != -1, FALSE);
+
+  OstreeRepoLockInfo info;
+  repo_lock_info (lock, &info);
+  g_return_val_if_fail (info.len > 0, FALSE);
+
+  g_debug ("Pop lock: state=%s, depth=%u", info.name, info.len);
+  if (info.len > 1)
+    {
+      int next_state = GPOINTER_TO_INT (g_queue_peek_nth (&lock->stack, 1));
+
+      /* Drop back to the previous lock state if it differs */
+      if (next_state != info.state)
+        {
+          /* We should never drop from shared to exclusive */
+          g_return_val_if_fail (next_state == LOCK_SH, FALSE);
+          g_debug ("Returning lock state to shared");
+          if (!do_repo_lock (lock->fd, next_state | flags))
+            return glnx_throw_errno_prefix (error,
+                                            "Setting repo lock to shared failed");
+        }
+      else
+        g_debug ("Maintaining lock state as %s", info.name);
+    }
+  else
+    {
+      /* Lock stack will be empty, unlock */
+      g_debug ("Unlocking repo");
+      if (!do_repo_unlock (lock->fd, flags))
+        return glnx_throw_errno_prefix (error, "Unlocking repo failed");
+    }
+
+  g_queue_pop_head (&lock->stack);
+
+  return TRUE;
+}
+
+/**
+ * ostree_repo_lock_push:
+ * @self: a #OstreeRepo
+ * @lock_type: the type of lock to acquire
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Takes a lock on the repository and adds it to the lock stack. If @lock_type
+ * is %OSTREE_REPO_LOCK_SHARED, a shared lock is taken. If @lock_type is
+ * %OSTREE_REPO_LOCK_EXCLUSIVE, an exclusive lock is taken. The actual lock
+ * state is only changed when locking a previously unlocked repository or
+ * upgrading the lock from shared to exclusive. If the requested lock state is
+ * unchanged or would represent a downgrade (exclusive to shared), the lock
+ * state is not changed and the stack is simply updated.
+ *
+ * ostree_repo_lock_push() waits for the lock depending on the repository's
+ * lock-timeout configuration. When lock-timeout is -1, a blocking lock is
+ * attempted. Otherwise, the lock is taken non-blocking and
+ * ostree_repo_lock_push() will sleep synchronously up to lock-timeout seconds
+ * attempting to acquire the lock. If the lock cannot be acquired within the
+ * timeout, a %G_IO_ERROR_WOULD_BLOCK error is returned.
+ *
+ * If @self is not writable by the user, then no locking is attempted and
+ * %TRUE is returned.
+ *
+ * Returns: %TRUE on success, otherwise %FALSE with @error set
+ * Since: 2017.14
+ */
+gboolean
+ostree_repo_lock_push (OstreeRepo          *self,
+                       OstreeRepoLockType   lock_type,
+                       GCancellable        *cancellable,
+                       GError             **error)
+{
+  g_return_val_if_fail (self != NULL, FALSE);
+  g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
+  g_return_val_if_fail (self->inited, FALSE);
+  g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE);
+  g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
+
+  if (!self->writable)
+    return TRUE;
+
+  g_assert (self->lock_timeout_seconds >= REPO_LOCK_DISABLED);
+  if (self->lock_timeout_seconds == REPO_LOCK_DISABLED)
+    return TRUE; /* No locking */
+  else if (self->lock_timeout_seconds == REPO_LOCK_BLOCKING)
+    {
+      g_debug ("Pushing lock blocking");
+      return push_repo_lock (self, lock_type, TRUE, error);
+    }
+  else
+    {
+      /* Convert to unsigned to guard against negative values */
+      guint lock_timeout_seconds = self->lock_timeout_seconds;
+      guint waited = 0;
+      g_debug ("Pushing lock non-blocking with timeout %u",
+               lock_timeout_seconds);
+      for (;;)
+        {
+          if (g_cancellable_set_error_if_cancelled (cancellable, error))
+            return FALSE;
+
+          g_autoptr(GError) local_error = NULL;
+          if (push_repo_lock (self, lock_type, FALSE, &local_error))
+            return TRUE;
+
+          if (!g_error_matches (local_error, G_IO_ERROR,
+                                G_IO_ERROR_WOULD_BLOCK))
+            {
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          if (waited >= lock_timeout_seconds)
+            {
+              g_debug ("Push lock: Could not acquire lock within %u seconds",
+                       lock_timeout_seconds);
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          /* Sleep 1 second and try again */
+          if (waited % 60 == 0)
+            {
+              guint remaining = lock_timeout_seconds - waited;
+              g_debug ("Push lock: Waiting %u more second%s to acquire lock",
+                       remaining, (remaining == 1) ? "" : "s");
+            }
+          waited++;
+          sleep (1);
+        }
+    }
+}
+
+/**
+ * ostree_repo_lock_pop:
+ * @self: a #OstreeRepo
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Remove the current repository lock state from the lock stack. If the lock
+ * stack becomes empty, the repository is unlocked. Otherwise, the lock state
+ * only changes when transitioning from an exclusive lock back to a shared
+ * lock.
+ *
+ * ostree_repo_lock_pop() waits for the lock depending on the repository's
+ * lock-timeout configuration. When lock-timeout is -1, a blocking lock is
+ * attempted. Otherwise, the lock is removed non-blocking and
+ * ostree_repo_lock_pop() will sleep synchronously up to lock-timeout seconds
+ * attempting to remove the lock. If the lock cannot be removed within the
+ * timeout, a %G_IO_ERROR_WOULD_BLOCK error is returned.
+ *
+ * If @self is not writable by the user, then no unlocking is attempted and
+ * %TRUE is returned.
+ *
+ * Returns: %TRUE on success, otherwise %FALSE with @error set
+ * Since: 2017.14
+ */
+gboolean
+ostree_repo_lock_pop (OstreeRepo    *self,
+                      GCancellable  *cancellable,
+                      GError       **error)
+{
+  g_return_val_if_fail (self != NULL, FALSE);
+  g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
+  g_return_val_if_fail (self->inited, FALSE);
+  g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE);
+  g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
+
+  if (!self->writable)
+    return TRUE;
+
+  g_assert (self->lock_timeout_seconds >= REPO_LOCK_DISABLED);
+  if (self->lock_timeout_seconds == REPO_LOCK_DISABLED)
+    return TRUE;
+  else if (self->lock_timeout_seconds == REPO_LOCK_BLOCKING)
+    {
+      g_debug ("Popping lock blocking");
+      return pop_repo_lock (self, TRUE, error);
+    }
+  else
+    {
+      /* Convert to unsigned to guard against negative values */
+      guint lock_timeout_seconds = self->lock_timeout_seconds;
+      guint waited = 0;
+      g_debug ("Popping lock non-blocking with timeout %u",
+               lock_timeout_seconds);
+      for (;;)
+        {
+          if (g_cancellable_set_error_if_cancelled (cancellable, error))
+            return FALSE;
+
+          g_autoptr(GError) local_error = NULL;
+          if (pop_repo_lock (self, FALSE, &local_error))
+            return TRUE;
+
+          if (!g_error_matches (local_error, G_IO_ERROR,
+                                G_IO_ERROR_WOULD_BLOCK))
+            {
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          if (waited >= lock_timeout_seconds)
+            {
+              g_debug ("Pop lock: Could not remove lock within %u seconds",
+                       lock_timeout_seconds);
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          /* Sleep 1 second and try again */
+          if (waited % 60 == 0)
+            {
+              guint remaining = lock_timeout_seconds - waited;
+              g_debug ("Pop lock: Waiting %u more second%s to remove lock",
+                       remaining, (remaining == 1) ? "" : "s");
+            }
+          waited++;
+          sleep (1);
+        }
+    }
+}
+
+/**
+ * ostree_repo_auto_lock_push: (skip)
+ * @self: a #OstreeRepo
+ * @lock_type: the type of lock to acquire
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Like ostree_repo_lock_push(), but for usage with #OstreeRepoAutoLock.
+ * The intended usage is to declare the #OstreeRepoAutoLock with
+ * g_autoptr() so that ostree_repo_auto_lock_cleanup() is called when it
+ * goes out of scope. This will automatically pop the lock status off
+ * the stack if it was acquired successfully.
+ *
+ * |[<!-- language="C" -->
+ * g_autoptr(OstreeRepoAutoLock) lock = NULL;
+ * lock = ostree_repo_auto_lock_push (repo, lock_type, cancellable, error);
+ * if (!lock)
+ *   return FALSE;
+ * ]|
+ *
+ * Returns: @self on success, otherwise %NULL with @error set
+ * Since: 2017.14
+ */
+OstreeRepoAutoLock *
+ostree_repo_auto_lock_push (OstreeRepo          *self,
+                            OstreeRepoLockType   lock_type,
+                            GCancellable        *cancellable,
+                            GError             **error)
+{
+  if (!ostree_repo_lock_push (self, lock_type, cancellable, error))
+    return NULL;
+  return (OstreeRepoAutoLock *)self;
+}
+
+/**
+ * ostree_repo_auto_lock_cleanup: (skip)
+ * @lock: a #OstreeRepoAutoLock
+ *
+ * A cleanup handler for use with ostree_repo_auto_lock_push(). If @lock is
+ * not %NULL, ostree_repo_lock_pop() will be called on it. If
+ * ostree_repo_lock_pop() fails, a critical warning will be emitted.
+ *
+ * Since: 2017.14
+ */
+void
+ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock)
+{
+  OstreeRepo *repo = lock;
+  if (repo)
+    {
+      g_autoptr(GError) error = NULL;
+      int errsv = errno;
+
+      if (!ostree_repo_lock_pop (repo, NULL, &error))
+        g_critical ("Cleanup repo lock failed: %s", error->message);
+
+      errno = errsv;
+    }
+}
+
 static GFile *
 get_remotes_d_dir (OstreeRepo          *self,
                    GFile               *sysroot);
@@ -505,18 +1028,26 @@ ostree_repo_finalize (GObject *object)
     g_hash_table_destroy (self->updated_uncompressed_dirs);
   if (self->config)
     g_key_file_free (self->config);
-  g_clear_pointer (&self->txn_refs, g_hash_table_destroy);
-  g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.collection_refs, g_hash_table_destroy);
   g_clear_error (&self->writable_error);
   g_clear_pointer (&self->object_sizes, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&self->dirmeta_cache, (GDestroyNotify) g_hash_table_unref);
   g_mutex_clear (&self->cache_lock);
-  g_mutex_clear (&self->txn_stats_lock);
+  g_mutex_clear (&self->txn_lock);
   g_free (self->collection_id);
 
   g_clear_pointer (&self->remotes, g_hash_table_destroy);
   g_mutex_clear (&self->remotes_lock);
 
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  if (lock_table)
+    {
+      g_hash_table_remove (lock_table, self);
+      if (g_hash_table_size (lock_table) == 0)
+        g_private_replace (&repo_lock_table, NULL);
+    }
+
   G_OBJECT_CLASS (ostree_repo_parent_class)->finalize (object);
 }
 
@@ -672,7 +1203,7 @@ ostree_repo_init (OstreeRepo *self)
                                                  test_error_keys, G_N_ELEMENTS (test_error_keys));
 
   g_mutex_init (&self->cache_lock);
-  g_mutex_init (&self->txn_stats_lock);
+  g_mutex_init (&self->txn_lock);
 
   self->remotes = g_hash_table_new_full (g_str_hash, g_str_equal,
                                          (GDestroyNotify) NULL,
@@ -1266,7 +1797,7 @@ _ostree_repo_remote_list (OstreeRepo *self,
   g_mutex_unlock (&self->remotes_lock);
 
   if (self->parent_repo)
-    _ostree_repo_remote_list (self, out);
+    _ostree_repo_remote_list (self->parent_repo, out);
 }
 
 /**
@@ -1797,6 +2328,7 @@ repo_create_at_internal (int             dfd,
                          GCancellable   *cancellable,
                          GError        **error)
 {
+  GLNX_AUTO_PREFIX_ERROR ("Creating repo", error);
    struct stat stbuf;
   /* We do objects/ last - if it exists we do nothing and exit successfully */
   const char *state_dirs[] = { "tmp", "extensions", "state",
@@ -2208,6 +2740,27 @@ reload_core_config (OstreeRepo          *self,
     self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10);
   }
 
+  /* Disable locking by default for now */
+  { gboolean locking;
+    if (!ot_keyfile_get_boolean_with_default (self->config, "core", "locking",
+                                              FALSE, &locking, error))
+      return FALSE;
+    if (!locking)
+      {
+        self->lock_timeout_seconds = REPO_LOCK_DISABLED;
+      }
+    else
+      {
+        g_autofree char *lock_timeout_seconds = NULL;
+
+        if (!ot_keyfile_get_value_with_default (self->config, "core", "lock-timeout-secs", "30",
+                                                &lock_timeout_seconds, error))
+          return FALSE;
+
+        self->lock_timeout_seconds = g_ascii_strtoull (lock_timeout_seconds, NULL, 10);
+      }
+  }
+
   { g_autofree char *compression_level_str = NULL;
 
     /* gzip defaults to 6 */
@@ -3388,6 +3941,115 @@ ostree_repo_delete_object (OstreeRepo           *self,
   return TRUE;
 }
 
+static gboolean
+fsck_metadata_object (OstreeRepo           *self,
+                      OstreeObjectType      objtype,
+                      const char           *sha256,
+                      GCancellable         *cancellable,
+                      GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck ", sha256, ".", ostree_object_type_to_string (objtype));
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GVariant) metadata = NULL;
+  if (!load_metadata_internal (self, objtype, sha256, TRUE,
+                               &metadata, NULL, NULL, NULL,
+                               cancellable, error))
+    return FALSE;
+
+  g_auto(OtChecksum) hasher = { 0, };
+  ot_checksum_init (&hasher);
+  ot_checksum_update (&hasher, g_variant_get_data (metadata), g_variant_get_size (metadata));
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&hasher, actual_checksum, sizeof (actual_checksum));
+  if (!_ostree_compare_object_checksum (objtype, sha256, actual_checksum, error))
+    return FALSE;
+
+  switch (objtype)
+    {
+    case OSTREE_OBJECT_TYPE_COMMIT:
+      if (!ostree_validate_structureof_commit (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_TREE:
+      if (!ostree_validate_structureof_dirtree (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_META:
+      if (!ostree_validate_structureof_dirmeta (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT:
+    case OSTREE_OBJECT_TYPE_COMMIT_META:
+      /* TODO */
+      break;
+    case OSTREE_OBJECT_TYPE_FILE:
+      g_assert_not_reached ();
+      break;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+fsck_content_object (OstreeRepo           *self,
+                     const char           *sha256,
+                     GCancellable         *cancellable,
+                     GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck content object ", sha256);
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GInputStream) input = NULL;
+  g_autoptr(GFileInfo) file_info = NULL;
+  g_autoptr(GVariant) xattrs = NULL;
+
+  if (!ostree_repo_load_file (self, sha256, &input, &file_info, &xattrs,
+                              cancellable, error))
+    return FALSE;
+
+  /* TODO more consistency checks here */
+  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  if (!ostree_validate_structureof_file_mode (mode, error))
+    return FALSE;
+
+  g_autofree guchar *computed_csum = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, input,
+                                        OSTREE_OBJECT_TYPE_FILE, &computed_csum,
+                                        cancellable, error))
+    return FALSE;
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ostree_checksum_inplace_from_bytes (computed_csum, actual_checksum);
+  return _ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE, sha256, actual_checksum, error);
+}
+
+/**
+ * ostree_repo_fsck_object:
+ * @self: Repo
+ * @objtype: Object type
+ * @sha256: Checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Verify consistency of the object; this performs checks only relevant to the
+ * immediate object itself, such as checksumming. This API call will not itself
+ * traverse metadata objects for example.
+ *
+ * Since: 2017.15
+ */
+gboolean
+ostree_repo_fsck_object (OstreeRepo           *self,
+                         OstreeObjectType      objtype,
+                         const char           *sha256,
+                         GCancellable         *cancellable,
+                         GError              **error)
+{
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    return fsck_metadata_object (self, objtype, sha256, cancellable, error);
+  else
+    return fsck_content_object (self, sha256, cancellable, error);
+}
+
 /**
  * ostree_repo_import_object_from:
  * @self: Destination repo

src/libostree/ostree-repo.h

@@ -107,6 +107,48 @@ OstreeRepo *  ostree_repo_create_at (int             dfd,
 
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
 
+/**
+ * OstreeRepoLockType:
+ * @OSTREE_REPO_LOCK_SHARED: A shared lock
+ * @OSTREE_REPO_LOCK_EXCLUSIVE: An exclusive lock
+ *
+ * The type of repository lock to acquire.
+ *
+ * Since: 2017.14
+ */
+typedef enum {
+  OSTREE_REPO_LOCK_SHARED,
+  OSTREE_REPO_LOCK_EXCLUSIVE
+} OstreeRepoLockType;
+
+_OSTREE_PUBLIC
+gboolean      ostree_repo_lock_push (OstreeRepo          *self,
+                                     OstreeRepoLockType   lock_type,
+                                     GCancellable        *cancellable,
+                                     GError             **error);
+_OSTREE_PUBLIC
+gboolean      ostree_repo_lock_pop (OstreeRepo    *self,
+                                    GCancellable  *cancellable,
+                                    GError       **error);
+
+/**
+ * OstreeRepoAutoLock: (skip)
+ *
+ * This is simply an alias to #OstreeRepo used for automatic lock cleanup.
+ * See ostree_repo_auto_lock_push() for its intended usage.
+ *
+ * Since: 2017.14
+ */
+typedef OstreeRepo OstreeRepoAutoLock;
+
+_OSTREE_PUBLIC
+OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo          *self,
+                                                 OstreeRepoLockType   lock_type,
+                                                 GCancellable        *cancellable,
+                                                 GError             **error);
+_OSTREE_PUBLIC
+void          ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock);
+
 _OSTREE_PUBLIC
 const gchar * ostree_repo_get_collection_id (OstreeRepo   *self);
 _OSTREE_PUBLIC
@@ -320,6 +362,12 @@ gboolean      ostree_repo_abort_transaction (OstreeRepo     *self,
                                              GCancellable   *cancellable,
                                              GError        **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_mark_commit_partial (OstreeRepo     *self,
+                                               const char     *checksum,
+                                               gboolean        is_partial,
+                                               GError        **error);
+
 _OSTREE_PUBLIC
 void          ostree_repo_transaction_set_refspec (OstreeRepo *self,
                                                    const char *refspec,
@@ -608,10 +656,17 @@ gboolean      ostree_repo_import_object_from_with_trust (OstreeRepo           *s
 _OSTREE_PUBLIC
 gboolean      ostree_repo_delete_object (OstreeRepo           *self,
                                          OstreeObjectType      objtype,
-                                         const char           *sha256, 
+                                         const char           *sha256,
                                          GCancellable         *cancellable,
                                          GError              **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_fsck_object (OstreeRepo           *self,
+                                       OstreeObjectType      objtype,
+                                       const char           *sha256,
+                                       GCancellable         *cancellable,
+                                       GError              **error);
+
 /** 
  * OstreeRepoCommitFilterResult:
  * @OSTREE_REPO_COMMIT_FILTER_ALLOW: Do commit this object

src/libostree/ostree-version.h

@@ -43,7 +43,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_RELEASE_VERSION (14)
+#define OSTREE_RELEASE_VERSION (15)
 
 /**
  * OSTREE_VERSION
@@ -52,7 +52,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION (2017.14)
+#define OSTREE_VERSION (2017.15)
 
 /**
  * OSTREE_VERSION_S:
@@ -62,7 +62,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION_S "2017.14"
+#define OSTREE_VERSION_S "2017.15"
 
 #define OSTREE_ENCODE_VERSION(year,release) \
         ((year) << 16 | (release))