Update upstream source from tag 'upstream/2017.15'

Update to upstream version '2017.15'
with Debian dir ef2f4fe701

Makefile-libostree.am

@@ -101,6 +101,7 @@ libostree_1_la_SOURCES = \
 	src/libostree/ostree-repo-checkout.c \
 	src/libostree/ostree-repo-commit.c \
 	src/libostree/ostree-repo-pull.c \
+	src/libostree/ostree-repo-pull-private.h \
 	src/libostree/ostree-repo-libarchive.c \
 	src/libostree/ostree-repo-prune.c \
 	src/libostree/ostree-repo-refs.c \

Makefile-tests.am

@@ -108,6 +108,7 @@ _installed_or_uninstalled_test_scripts = \
 	tests/test-xattrs.sh \
 	tests/test-auto-summary.sh \
 	tests/test-prune.sh \
+	tests/test-concurrency.py \
 	tests/test-refs.sh \
 	tests/test-demo-buildsystem.sh \
 	tests/test-switchroot.sh \

Makefile.am

@@ -31,7 +31,8 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \
   -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \
 	-DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 '-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,50)' \
 	-DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 '-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)'
-AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS)
+# For strict aliasing, see https://bugzilla.gnome.org/show_bug.cgi?id=791622
+AM_CFLAGS += -std=gnu99 -fno-strict-aliasing $(WARN_CFLAGS)
 AM_DISTCHECK_CONFIGURE_FLAGS += \
 	--enable-gtk-doc \
 	--enable-man \

Makefile.in

@@ -704,6 +704,7 @@ am__libostree_1_la_SOURCES_DIST =  \
 	src/libostree/ostree-repo-checkout.c \
 	src/libostree/ostree-repo-commit.c \
 	src/libostree/ostree-repo-pull.c \
+	src/libostree/ostree-repo-pull-private.h \
 	src/libostree/ostree-repo-libarchive.c \
 	src/libostree/ostree-repo-prune.c \
 	src/libostree/ostree-repo-refs.c \
@@ -1728,11 +1729,12 @@ am__EXEEXT_25 = tests/test-basic.sh tests/test-basic-user.sh \
 	tests/test-reset-nonlinear.sh tests/test-oldstyle-partial.sh \
 	tests/test-delta.sh tests/test-xattrs.sh \
 	tests/test-auto-summary.sh tests/test-prune.sh \
-	tests/test-refs.sh tests/test-demo-buildsystem.sh \
+	tests/test-concurrency.py tests/test-refs.sh \
-	tests/test-switchroot.sh tests/test-pull-contenturl.sh \
+	tests/test-demo-buildsystem.sh tests/test-switchroot.sh \
-	tests/test-pull-mirrorlist.sh tests/test-summary-update.sh \
+	tests/test-pull-contenturl.sh tests/test-pull-mirrorlist.sh \
-	tests/test-summary-view.sh $(am__EXEEXT_2) $(am__EXEEXT_22) \
+	tests/test-summary-update.sh tests/test-summary-view.sh \
-	$(am__append_68) $(am__append_71) $(am__EXEEXT_24)
+	$(am__EXEEXT_2) $(am__EXEEXT_22) $(am__append_68) \
+	$(am__append_71) $(am__EXEEXT_24)
 @ENABLE_INSTALLED_TESTS_EXCLUSIVE_FALSE@am__EXEEXT_26 =  \
 @ENABLE_INSTALLED_TESTS_EXCLUSIVE_FALSE@	$(am__EXEEXT_25)
 am__EXEEXT_27 = $(am__EXEEXT_2) $(am__EXEEXT_26)
@@ -2023,7 +2025,8 @@ AM_CPPFLAGS = -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \
 	'-DGLIB_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,50)' \
 	-DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 \
 	'-DSOUP_VERSION_MAX_ALLOWED=G_ENCODE_VERSION(2,48)'
-AM_CFLAGS = -std=gnu99 $(WARN_CFLAGS)
+# For strict aliasing, see https://bugzilla.gnome.org/show_bug.cgi?id=791622
+AM_CFLAGS = -std=gnu99 -fno-strict-aliasing $(WARN_CFLAGS)
 
 # Allow the distcheck install under $prefix test to pass
 AM_DISTCHECK_CONFIGURE_FLAGS = --enable-gtk-doc --enable-man \
@@ -2315,6 +2318,7 @@ libostree_1_la_SOURCES = src/libostree/ostree-async-progress.c \
 	src/libostree/ostree-repo-checkout.c \
 	src/libostree/ostree-repo-commit.c \
 	src/libostree/ostree-repo-pull.c \
+	src/libostree/ostree-repo-pull-private.h \
 	src/libostree/ostree-repo-libarchive.c \
 	src/libostree/ostree-repo-prune.c \
 	src/libostree/ostree-repo-refs.c \
@@ -2547,11 +2551,12 @@ _installed_or_uninstalled_test_scripts = tests/test-basic.sh \
 	tests/test-reset-nonlinear.sh tests/test-oldstyle-partial.sh \
 	tests/test-delta.sh tests/test-xattrs.sh \
 	tests/test-auto-summary.sh tests/test-prune.sh \
-	tests/test-refs.sh tests/test-demo-buildsystem.sh \
+	tests/test-concurrency.py tests/test-refs.sh \
-	tests/test-switchroot.sh tests/test-pull-contenturl.sh \
+	tests/test-demo-buildsystem.sh tests/test-switchroot.sh \
-	tests/test-pull-mirrorlist.sh tests/test-summary-update.sh \
+	tests/test-pull-contenturl.sh tests/test-pull-mirrorlist.sh \
-	tests/test-summary-view.sh $(NULL) $(am__append_65) \
+	tests/test-summary-update.sh tests/test-summary-view.sh \
-	$(am__append_68) $(am__append_71) $(am__append_72)
+	$(NULL) $(am__append_65) $(am__append_68) $(am__append_71) \
+	$(am__append_72)
 experimental_test_scripts = \
 	tests/test-create-usb.sh \
 	tests/test-find-remotes.sh \
@@ -7666,6 +7671,13 @@ tests/test-prune.sh.log: tests/test-prune.sh
 	--log-file $$b.log --trs-file $$b.trs \
 	$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
 	"$$tst" $(AM_TESTS_FD_REDIRECT)
+tests/test-concurrency.py.log: tests/test-concurrency.py
+	@p='tests/test-concurrency.py'; \
+	b='tests/test-concurrency.py'; \
+	$(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \
+	--log-file $$b.log --trs-file $$b.trs \
+	$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
+	"$$tst" $(AM_TESTS_FD_REDIRECT)
 tests/test-refs.sh.log: tests/test-refs.sh
 	@p='tests/test-refs.sh'; \
 	b='tests/test-refs.sh'; \

README.md

@@ -70,6 +70,10 @@ projects.
 where OSTree was born - as a high performance continuous delivery/testing
 system for GNOME.
 
+The [BuildStream](https://gitlab.com/BuildStream/buildstream) build and
+integration tool uses libostree as a caching system to store and share
+built artifacts.
+
 Building
 --------
 

apidoc/Makefile.am

@@ -83,6 +83,7 @@ IGNORE_HFILES= \
 	ostree-metalink.h \
 	ostree-repo-file-enumerator.h \
 	ostree-repo-private.h \
+	ostree-repo-pull-private.h \
 	ostree-repo-static-delta-private.h \
 	ostree-sysroot-private.h \
 	ostree-tls-cert-interaction.h \

apidoc/Makefile.in

@@ -453,6 +453,7 @@ IGNORE_HFILES = \
 	ostree-metalink.h \
 	ostree-repo-file-enumerator.h \
 	ostree-repo-private.h \
+	ostree-repo-pull-private.h \
 	ostree-repo-static-delta-private.h \
 	ostree-sysroot-private.h \
 	ostree-tls-cert-interaction.h \

apidoc/html/index.html

@@ -14,7 +14,7 @@
 <div class="titlepage">
 <div>
 <div><table class="navigation" id="top" width="100%" cellpadding="2" cellspacing="0"><tr><th valign="middle"><p class="title">OSTree API references</p></th></tr></table></div>
-<div><p class="releaseinfo">for OSTree 2017.14</p></div>
+<div><p class="releaseinfo">for OSTree 2017.15</p></div>
 </div>
 <hr>
 </div>

apidoc/html/ostree-Core-repository-independent-functions.html

@@ -294,6 +294,14 @@
 <span class="returnvalue">gboolean</span>
 </td>
 <td class="function_name">
+<a class="link" href="ostree-Core-repository-independent-functions.html#ostree-break-hardlink" title="ostree_break_hardlink ()">ostree_break_hardlink</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<span class="returnvalue">gboolean</span>
+</td>
+<td class="function_name">
 <a class="link" href="ostree-Core-repository-independent-functions.html#ostree-checksum-file-from-input" title="ostree_checksum_file_from_input ()">ostree_checksum_file_from_input</a> <span class="c_punctuation">()</span>
 </td>
 </tr>
@@ -1686,6 +1694,61 @@ for writing data to an <a class="link" href="ostree-OstreeRepo.html#OstreeRepo"
 </div>
 <hr>
 <div class="refsect2">
+<a name="ostree-break-hardlink"></a><h3>ostree_break_hardlink ()</h3>
+<pre class="programlisting"><span class="returnvalue">gboolean</span>
+ostree_break_hardlink (<em class="parameter"><code><span class="type">int</span> dfd</code></em>,
+                       <em class="parameter"><code>const <span class="type">char</span> *path</code></em>,
+                       <em class="parameter"><code><span class="type">gboolean</span> skip_xattrs</code></em>,
+                       <em class="parameter"><code><span class="type">GCancellable</span> *cancellable</code></em>,
+                       <em class="parameter"><code><span class="type">GError</span> **error</code></em>);</pre>
+<p>In many cases using libostree, a program may need to "break"
+hardlinks by performing a copy.  For example, in order to
+logically append to a file.</p>
+<p>This function performs full copying, including e.g. extended
+attributes and permissions of both regular files and symbolic links.</p>
+<p>If the file is not hardlinked, this function does nothing and
+returns successfully.</p>
+<p>This function does not perform synchronization via <code class="literal"><code class="function">fsync()</code></code> or
+<code class="literal"><code class="function">fdatasync()</code></code>; the idea is this will commonly be done as part
+of an <code class="literal"><a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a></code>, which itself takes
+care of synchronization.</p>
+<div class="refsect3">
+<a name="ostree-break-hardlink.parameters"></a><h4>Parameters</h4>
+<div class="informaltable"><table class="informaltable" width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>dfd</p></td>
+<td class="parameter_description"><p>Directory fd</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>path</p></td>
+<td class="parameter_description"><p>Path relative to <em class="parameter"><code>dfd</code></em>
+</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>skip_xattrs</p></td>
+<td class="parameter_description"><p>Do not copy extended attributes</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p>error</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<p class="since">Since: 2017.15</p>
+</div>
+<hr>
+<div class="refsect2">
 <a name="ostree-checksum-file-from-input"></a><h3>ostree_checksum_file_from_input ()</h3>
 <pre class="programlisting"><span class="returnvalue">gboolean</span>
 ostree_checksum_file_from_input (<em class="parameter"><code><span class="type">GFileInfo</span> *file_info</code></em>,

apidoc/html/ostree-OstreeRepo.html

@@ -412,6 +412,14 @@
 <span class="returnvalue">gboolean</span>
 </td>
 <td class="function_name">
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-mark-commit-partial" title="ostree_repo_mark_commit_partial ()">ostree_repo_mark_commit_partial</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
+<span class="returnvalue">gboolean</span>
+</td>
+<td class="function_name">
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-write-metadata" title="ostree_repo_write_metadata ()">ostree_repo_write_metadata</a> <span class="c_punctuation">()</span>
 </td>
 </tr>
@@ -609,6 +617,14 @@
 </tr>
 <tr>
 <td class="function_type">
+<span class="returnvalue">gboolean</span>
+</td>
+<td class="function_name">
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-fsck-object" title="ostree_repo_fsck_object ()">ostree_repo_fsck_object</a> <span class="c_punctuation">()</span>
+</td>
+</tr>
+<tr>
+<td class="function_type">
 <a class="link" href="ostree-OstreeRepo.html#OstreeRepoCommitFilterResult" title="enum OstreeRepoCommitFilterResult"><span class="returnvalue">OstreeRepoCommitFilterResult</span></a>
 </td>
 <td class="function_name">
@@ -2644,6 +2660,7 @@ entire objects directory. If your commit is composed of mostly hardlinks to
 existing ostree objects, then this will speed up considerably, so call it
 before you call <code class="function">ostree_write_directory_to_mtree()</code> or similar.  However,
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-devino-cache-new" title="ostree_repo_devino_cache_new ()"><code class="function">ostree_repo_devino_cache_new()</code></a> is better as it avoids scanning all objects.</p>
+<p>Multithreading: This function is *not* MT safe.</p>
 <div class="refsect3">
 <a name="ostree-repo-scan-hardlinks.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -2684,8 +2701,17 @@ ostree_repo_prepare_transaction (<em class="parameter"><code><a class="link" hre
 need to start a transaction. You can complete the transaction with
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a>, or abort the transaction with
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-abort-transaction" title="ostree_repo_abort_transaction ()"><code class="function">ostree_repo_abort_transaction()</code></a>.</p>
-<p>Currently, transactions are not atomic, and aborting a transaction
+<p>Currently, transactions may result in partial commits or data in the target
-will not erase any data you  write during the transaction.</p>
+repository if interrupted during <a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a>, and
+further writing refs is also not currently atomic.</p>
+<p>There can be at most one transaction active on a repo at a time per instance
+of <code class="literal">OstreeRepo</code>; however, it is safe to have multiple threads writing objects
+on a single <code class="literal">OstreeRepo</code> instance as long as their lifetime is bounded by the
+transaction.</p>
+<p>Multithreading: This function is *not* MT safe; only one transaction can be
+active at a time.</p>
+<p>This function takes a shared lock on the <em class="parameter"><code>self</code></em>
+ repository.</p>
 <div class="refsect3">
 <a name="ostree-repo-prepare-transaction.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -2732,6 +2758,10 @@ ostree_repo_commit_transaction (<em class="parameter"><code><a class="link" href
 <p>Complete the transaction. Any refs set with
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-transaction-set-ref" title="ostree_repo_transaction_set_ref ()"><code class="function">ostree_repo_transaction_set_ref()</code></a> or
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-transaction-set-refspec" title="ostree_repo_transaction_set_refspec ()"><code class="function">ostree_repo_transaction_set_refspec()</code></a> will be written out.</p>
+<p>Note that if multiple threads are performing writes, all such threads must
+have terminated before this function is invoked.</p>
+<p>Multithreading: This function is *not* MT safe; only one transaction can be
+active at a time.</p>
 <div class="refsect3">
 <a name="ostree-repo-commit-transaction.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -2816,6 +2846,7 @@ ostree_repo_transaction_set_refspec (<em class="parameter"><code><a class="link"
 <em class="parameter"><code>refspec</code></em>
  format as input instead of separate remote and name
 arguments.</p>
+<p>Multithreading: Since v2017.15 this function is MT safe.</p>
 <div class="refsect3">
 <a name="ostree-repo-transaction-set-refspec.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -2861,10 +2892,19 @@ remote.</p>
 <p>Otherwise, if <em class="parameter"><code>checksum</code></em>
  is <code class="literal">NULL</code>, then record that the ref should
 be deleted.</p>
-<p>The change will not be written out immediately, but when the transaction
+<p>The change will be written when the transaction is completed with
-is completed with <a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a>. If the transaction
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a>; that function takes care of writing all of
-is instead aborted with <a class="link" href="ostree-OstreeRepo.html#ostree-repo-abort-transaction" title="ostree_repo_abort_transaction ()"><code class="function">ostree_repo_abort_transaction()</code></a>, no changes will
+the objects (such as the commit referred to by <em class="parameter"><code>checksum</code></em>
-be made to the repository.</p>
+) before updating the
+refs. If the transaction is instead aborted with
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-abort-transaction" title="ostree_repo_abort_transaction ()"><code class="function">ostree_repo_abort_transaction()</code></a>, no changes to the ref will be made to the
+repository.</p>
+<p>Note however that currently writing *multiple* refs is not truly atomic; if
+the process or system is terminated during
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-commit-transaction" title="ostree_repo_commit_transaction ()"><code class="function">ostree_repo_commit_transaction()</code></a>, it is possible that just some of the refs
+will have been updated. Your application should take care to handle this
+case.</p>
+<p>Multithreading: Since v2017.15 this function is MT safe.</p>
 <div class="refsect3">
 <a name="ostree-repo-transaction-set-ref.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -2911,6 +2951,7 @@ ostree_repo_set_ref_immediate (<em class="parameter"><code><a class="link" href=
 <p>This is like <a class="link" href="ostree-OstreeRepo.html#ostree-repo-transaction-set-ref" title="ostree_repo_transaction_set_ref ()"><code class="function">ostree_repo_transaction_set_ref()</code></a>, except it may be
 invoked outside of a transaction.  This is presently safe for the
 case where we're creating or overwriting an existing ref.</p>
+<p>Multithreading: This function is MT safe.</p>
 <div class="refsect3">
 <a name="ostree-repo-set-ref-immediate.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -3184,6 +3225,53 @@ ostree_repo_has_object (<em class="parameter"><code><a class="link" href="ostree
 </div>
 <hr>
 <div class="refsect2">
+<a name="ostree-repo-mark-commit-partial"></a><h3>ostree_repo_mark_commit_partial ()</h3>
+<pre class="programlisting"><span class="returnvalue">gboolean</span>
+ostree_repo_mark_commit_partial (<em class="parameter"><code><a class="link" href="ostree-OstreeRepo.html#OstreeRepo" title="OstreeRepo"><span class="type">OstreeRepo</span></a> *self</code></em>,
+                                 <em class="parameter"><code>const <span class="type">char</span> *checksum</code></em>,
+                                 <em class="parameter"><code><span class="type">gboolean</span> is_partial</code></em>,
+                                 <em class="parameter"><code><span class="type">GError</span> **error</code></em>);</pre>
+<p>Commits in "partial" state do not have all their child objects written. This
+occurs in various situations, such as during a pull, but also if a "subpath"
+pull is used, as well as "commit only" pulls.</p>
+<p>This function is used by <a class="link" href="ostree-OstreeRepo.html#ostree-repo-pull-with-options" title="ostree_repo_pull_with_options ()"><code class="function">ostree_repo_pull_with_options()</code></a>; you
+should use this if you are implementing a different type of transport.</p>
+<div class="refsect3">
+<a name="ostree-repo-mark-commit-partial.parameters"></a><h4>Parameters</h4>
+<div class="informaltable"><table class="informaltable" width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>self</p></td>
+<td class="parameter_description"><p>Repo</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>checksum</p></td>
+<td class="parameter_description"><p>Commit SHA-256</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>is_partial</p></td>
+<td class="parameter_description"><p>Whether or not this commit is partial</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p>Error</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<p class="since">Since: 2017.15</p>
+</div>
+<hr>
+<div class="refsect2">
 <a name="ostree-repo-write-metadata"></a><h3>ostree_repo_write_metadata ()</h3>
 <pre class="programlisting"><span class="returnvalue">gboolean</span>
 ostree_repo_write_metadata (<em class="parameter"><code><a class="link" href="ostree-OstreeRepo.html#OstreeRepo" title="OstreeRepo"><span class="type">OstreeRepo</span></a> *self</code></em>,
@@ -3836,6 +3924,11 @@ with their current values in <em class="parameter"><code>out_all_refs</code></em
 .  Otherwise, only list
 refspecs which have <em class="parameter"><code>refspec_prefix</code></em>
  as a prefix.</p>
+<p><em class="parameter"><code>out_all_refs</code></em>
+ will be returned as a mapping from refspecs (including the
+remote name) to checksums. If <em class="parameter"><code>refspec_prefix</code></em>
+ is non-<code class="literal">NULL</code>, it will be
+removed as a prefix from the hash table keys.</p>
 <div class="refsect3">
 <a name="ostree-repo-list-refs.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -3857,7 +3950,7 @@ refspecs which have <em class="parameter"><code>refspec_prefix</code></em>
 </tr>
 <tr>
 <td class="parameter_name"><p>out_all_refs</p></td>
-<td class="parameter_description"><p>Mapping from ref to checksum. </p></td>
+<td class="parameter_description"><p>Mapping from refspec to checksum. </p></td>
 <td class="parameter_annotations"><span class="annotation">[<a href="http://foldoc.org/out"><span class="acronym">out</span></a>][<a href="http://foldoc.org/element-type"><span class="acronym">element-type</span></a> utf8 utf8][<a href="http://foldoc.org/transfer%20container"><span class="acronym">transfer container</span></a>]</span></td>
 </tr>
 <tr>
@@ -3889,9 +3982,12 @@ ostree_repo_list_refs_ext (<em class="parameter"><code><a class="link" href="ost
 with their current values in <em class="parameter"><code>out_all_refs</code></em>
 .  Otherwise, only list
 refspecs which have <em class="parameter"><code>refspec_prefix</code></em>
- as a prefix.  Differently from
+ as a prefix.</p>
-<a class="link" href="ostree-OstreeRepo.html#ostree-repo-list-refs" title="ostree_repo_list_refs ()"><code class="function">ostree_repo_list_refs()</code></a>, the prefix will not be removed from the ref
+<p><em class="parameter"><code>out_all_refs</code></em>
-name.</p>
+ will be returned as a mapping from refspecs (including the
+remote name) to checksums. Differently from <a class="link" href="ostree-OstreeRepo.html#ostree-repo-list-refs" title="ostree_repo_list_refs ()"><code class="function">ostree_repo_list_refs()</code></a>, the
+<em class="parameter"><code>refspec_prefix</code></em>
+ will not be removed from the refspecs in the hash table.</p>
 <div class="refsect3">
 <a name="ostree-repo-list-refs-ext.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -3913,7 +4009,7 @@ name.</p>
 </tr>
 <tr>
 <td class="parameter_name"><p>out_all_refs</p></td>
-<td class="parameter_description"><p>Mapping from ref to checksum. </p></td>
+<td class="parameter_description"><p>Mapping from refspec to checksum. </p></td>
 <td class="parameter_annotations"><span class="annotation">[<a href="http://foldoc.org/out"><span class="acronym">out</span></a>][<a href="http://foldoc.org/element-type"><span class="acronym">element-type</span></a> utf8 utf8][<a href="http://foldoc.org/transfer%20container"><span class="acronym">transfer container</span></a>]</span></td>
 </tr>
 <tr>
@@ -4624,6 +4720,57 @@ is thrown if the object does not exist.</p>
 </div>
 <hr>
 <div class="refsect2">
+<a name="ostree-repo-fsck-object"></a><h3>ostree_repo_fsck_object ()</h3>
+<pre class="programlisting"><span class="returnvalue">gboolean</span>
+ostree_repo_fsck_object (<em class="parameter"><code><a class="link" href="ostree-OstreeRepo.html#OstreeRepo" title="OstreeRepo"><span class="type">OstreeRepo</span></a> *self</code></em>,
+                         <em class="parameter"><code><a class="link" href="ostree-Core-repository-independent-functions.html#OstreeObjectType" title="enum OstreeObjectType"><span class="type">OstreeObjectType</span></a> objtype</code></em>,
+                         <em class="parameter"><code>const <span class="type">char</span> *sha256</code></em>,
+                         <em class="parameter"><code><span class="type">GCancellable</span> *cancellable</code></em>,
+                         <em class="parameter"><code><span class="type">GError</span> **error</code></em>);</pre>
+<p>Verify consistency of the object; this performs checks only relevant to the
+immediate object itself, such as checksumming. This API call will not itself
+traverse metadata objects for example.</p>
+<div class="refsect3">
+<a name="ostree-repo-fsck-object.parameters"></a><h4>Parameters</h4>
+<div class="informaltable"><table class="informaltable" width="100%" border="0">
+<colgroup>
+<col width="150px" class="parameters_name">
+<col class="parameters_description">
+<col width="200px" class="parameters_annotations">
+</colgroup>
+<tbody>
+<tr>
+<td class="parameter_name"><p>self</p></td>
+<td class="parameter_description"><p>Repo</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>objtype</p></td>
+<td class="parameter_description"><p>Object type</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>sha256</p></td>
+<td class="parameter_description"><p>Checksum</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>cancellable</p></td>
+<td class="parameter_description"><p>Cancellable</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+<tr>
+<td class="parameter_name"><p>error</p></td>
+<td class="parameter_description"><p>Error</p></td>
+<td class="parameter_annotations"> </td>
+</tr>
+</tbody>
+</table></div>
+</div>
+<p class="since">Since: 2017.15</p>
+</div>
+<hr>
+<div class="refsect2">
 <a name="OstreeRepoCommitFilter"></a><h3>OstreeRepoCommitFilter ()</h3>
 <pre class="programlisting"><a class="link" href="ostree-OstreeRepo.html#OstreeRepoCommitFilterResult" title="enum OstreeRepoCommitFilterResult"><span class="returnvalue">OstreeRepoCommitFilterResult</span></a>
 <span class="c_punctuation">(</span>*OstreeRepoCommitFilter<span class="c_punctuation">)</span> (<em class="parameter"><code><a class="link" href="ostree-OstreeRepo.html#OstreeRepo" title="OstreeRepo"><span class="type">OstreeRepo</span></a> *repo</code></em>,
@@ -6436,6 +6583,8 @@ history from the repository.</p>
 <p>Use the <a class="link" href="ostree-OstreeRepo.html#OSTREE-REPO-PRUNE-FLAGS-NO-PRUNE:CAPS"><code class="literal">OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE</code></a> to just determine
 statistics on objects that would be deleted, without actually
 deleting them.</p>
+<p>This function takes an exclusive lock on the <em class="parameter"><code>self</code></em>
+ repository.</p>
 <div class="refsect3">
 <a name="ostree-repo-prune.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -6500,6 +6649,8 @@ ostree_repo_prune_static_deltas (<em class="parameter"><code><a class="link" hre
 <p>Prune static deltas, if COMMIT is specified then delete static delta files only
 targeting that commit; otherwise any static delta of non existing commits are
 deleted.</p>
+<p>This function takes an exclusive lock on the <em class="parameter"><code>self</code></em>
+ repository.</p>
 <div class="refsect3">
 <a name="ostree-repo-prune-static-deltas.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">
@@ -6554,6 +6705,8 @@ retain all commits from a production branch, but just GC some history from
 your dev branch.</p>
 <p>The <a class="link" href="ostree-OstreeRepo.html#OSTREE-REPO-PRUNE-FLAGS-NO-PRUNE:CAPS"><code class="literal">OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE</code></a> flag may be specified to just determine
 statistics on objects that would be deleted, without actually deleting them.</p>
+<p>This function takes an exclusive lock on the <em class="parameter"><code>self</code></em>
+ repository.</p>
 <div class="refsect3">
 <a name="ostree-repo-prune-from-reachable.parameters"></a><h4>Parameters</h4>
 <div class="informaltable"><table class="informaltable" width="100%" border="0">

apidoc/html/ostree.devhelp2

@@ -52,6 +52,7 @@
     <keyword type="function" name="ostree_raw_file_to_archive_z2_stream ()" link="ostree-Core-repository-independent-functions.html#ostree-raw-file-to-archive-z2-stream"/>
     <keyword type="function" name="ostree_raw_file_to_archive_z2_stream_with_options ()" link="ostree-Core-repository-independent-functions.html#ostree-raw-file-to-archive-z2-stream-with-options" since="2017.3"/>
     <keyword type="function" name="ostree_raw_file_to_content_stream ()" link="ostree-Core-repository-independent-functions.html#ostree-raw-file-to-content-stream"/>
+    <keyword type="function" name="ostree_break_hardlink ()" link="ostree-Core-repository-independent-functions.html#ostree-break-hardlink" since="2017.15"/>
     <keyword type="function" name="ostree_checksum_file_from_input ()" link="ostree-Core-repository-independent-functions.html#ostree-checksum-file-from-input"/>
     <keyword type="function" name="ostree_checksum_file ()" link="ostree-Core-repository-independent-functions.html#ostree-checksum-file"/>
     <keyword type="function" name="ostree_checksum_file_at ()" link="ostree-Core-repository-independent-functions.html#ostree-checksum-file-at" since="2017.13"/>
@@ -128,6 +129,7 @@
     <keyword type="function" name="ostree_repo_set_cache_dir ()" link="ostree-OstreeRepo.html#ostree-repo-set-cache-dir"/>
     <keyword type="function" name="ostree_repo_sign_delta ()" link="ostree-OstreeRepo.html#ostree-repo-sign-delta"/>
     <keyword type="function" name="ostree_repo_has_object ()" link="ostree-OstreeRepo.html#ostree-repo-has-object"/>
+    <keyword type="function" name="ostree_repo_mark_commit_partial ()" link="ostree-OstreeRepo.html#ostree-repo-mark-commit-partial" since="2017.15"/>
     <keyword type="function" name="ostree_repo_write_metadata ()" link="ostree-OstreeRepo.html#ostree-repo-write-metadata"/>
     <keyword type="function" name="ostree_repo_write_metadata_async ()" link="ostree-OstreeRepo.html#ostree-repo-write-metadata-async"/>
     <keyword type="function" name="ostree_repo_write_metadata_finish ()" link="ostree-OstreeRepo.html#ostree-repo-write-metadata-finish"/>
@@ -153,6 +155,7 @@
     <keyword type="function" name="ostree_repo_import_archive_to_mtree ()" link="ostree-OstreeRepo.html#ostree-repo-import-archive-to-mtree"/>
     <keyword type="function" name="ostree_repo_export_tree_to_archive ()" link="ostree-OstreeRepo.html#ostree-repo-export-tree-to-archive"/>
     <keyword type="function" name="ostree_repo_delete_object ()" link="ostree-OstreeRepo.html#ostree-repo-delete-object"/>
+    <keyword type="function" name="ostree_repo_fsck_object ()" link="ostree-OstreeRepo.html#ostree-repo-fsck-object" since="2017.15"/>
     <keyword type="function" name="OstreeRepoCommitFilter ()" link="ostree-OstreeRepo.html#OstreeRepoCommitFilter"/>
     <keyword type="function" name="ostree_repo_commit_modifier_new ()" link="ostree-OstreeRepo.html#ostree-repo-commit-modifier-new"/>
     <keyword type="function" name="OstreeRepoCommitModifierXattrCallback ()" link="ostree-OstreeRepo.html#OstreeRepoCommitModifierXattrCallback"/>

apidoc/html/reference.html

@@ -164,6 +164,10 @@
 <a class="link" href="ostree-ostree-bootconfig-parser.html#ostree-bootconfig-parser-write-at" title="ostree_bootconfig_parser_write_at ()">ostree_bootconfig_parser_write_at</a>, function in <a class="link" href="ostree-ostree-bootconfig-parser.html" title="ostree-bootconfig-parser">ostree-bootconfig-parser</a>
 </dt>
 <dd></dd>
+<dt>
+<a class="link" href="ostree-Core-repository-independent-functions.html#ostree-break-hardlink" title="ostree_break_hardlink ()">ostree_break_hardlink</a>, function in <a class="link" href="ostree-Core-repository-independent-functions.html" title="Core repository-independent functions">Core repository-independent functions</a>
+</dt>
+<dd></dd>
 <a name="idxC"></a><h3 class="title">C</h3>
 <dt>
 <a class="link" href="ostree-ostree-chain-input-stream.html#OstreeChainInputStream" title="struct OstreeChainInputStream">OstreeChainInputStream</a>, struct in <a class="link" href="ostree-ostree-chain-input-stream.html" title="ostree-chain-input-stream">ostree-chain-input-stream</a>
@@ -858,6 +862,10 @@ OSTREE_RELEASE_VERSION, macro in ostree-version
 </dt>
 <dd></dd>
 <dt>
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-fsck-object" title="ostree_repo_fsck_object ()">ostree_repo_fsck_object</a>, function in <a class="link" href="ostree-OstreeRepo.html" title="OstreeRepo: Content-addressed object store">OstreeRepo</a>
+</dt>
+<dd></dd>
+<dt>
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-get-config" title="ostree_repo_get_config ()">ostree_repo_get_config</a>, function in <a class="link" href="ostree-OstreeRepo.html" title="OstreeRepo: Content-addressed object store">OstreeRepo</a>
 </dt>
 <dd></dd>
@@ -970,6 +978,10 @@ OSTREE_RELEASE_VERSION, macro in ostree-version
 </dt>
 <dd></dd>
 <dt>
+<a class="link" href="ostree-OstreeRepo.html#ostree-repo-mark-commit-partial" title="ostree_repo_mark_commit_partial ()">ostree_repo_mark_commit_partial</a>, function in <a class="link" href="ostree-OstreeRepo.html" title="OstreeRepo: Content-addressed object store">OstreeRepo</a>
+</dt>
+<dd></dd>
+<dt>
 <a class="link" href="ostree-OstreeRepo.html#ostree-repo-mode-from-string" title="ostree_repo_mode_from_string ()">ostree_repo_mode_from_string</a>, function in <a class="link" href="ostree-OstreeRepo.html" title="OstreeRepo: Content-addressed object store">OstreeRepo</a>
 </dt>
 <dd></dd>

apidoc/ostree-experimental-sections.txt

@@ -90,6 +90,12 @@ ostree_repo_finder_override_get_type
 
 <SECTION>
 <FILE>ostree-misc-experimental</FILE>
+OstreeRepoLockType
+ostree_repo_lock_push
+ostree_repo_lock_pop
+OstreeRepoAutoLock
+ostree_repo_auto_lock_push
+ostree_repo_auto_lock_cleanup
 ostree_repo_get_collection_id
 ostree_repo_set_collection_id
 ostree_validate_collection_id

apidoc/ostree-sections.txt

@@ -133,6 +133,7 @@ ostree_content_file_parse_at
 ostree_raw_file_to_archive_z2_stream
 ostree_raw_file_to_archive_z2_stream_with_options
 ostree_raw_file_to_content_stream
+ostree_break_hardlink
 ostree_checksum_file_from_input
 ostree_checksum_file
 ostree_checksum_file_at
@@ -320,6 +321,7 @@ ostree_repo_set_alias_ref_immediate
 ostree_repo_set_cache_dir
 ostree_repo_sign_delta
 ostree_repo_has_object
+ostree_repo_mark_commit_partial
 ostree_repo_write_metadata
 ostree_repo_write_metadata_async
 ostree_repo_write_metadata_finish
@@ -348,6 +350,7 @@ ostree_repo_import_object_from_with_trust
 ostree_repo_import_archive_to_mtree
 ostree_repo_export_tree_to_archive
 ostree_repo_delete_object
+ostree_repo_fsck_object
 OstreeRepoCommitFilterResult
 OstreeRepoCommitFilter
 OstreeRepoCommitModifier

apidoc/version.xml

@@ -1 +1 @@
-2017.14
+2017.15

bash/ostree

@@ -963,6 +963,8 @@ _ostree_fsck() {
         --add-tombstones
         --delete
         --quiet -q
+        --verify-bindings
+        --verify-back-refs
     "
 
     local options_with_args="

config.h.in

@@ -6,6 +6,9 @@
 /* Define if we are enabling ostree trivial-httpd entrypoint */
 #undef BUILDOPT_ENABLE_TRIVIAL_HTTPD_CMDLINE
 
+/* Define if we enable http2 */
+#undef BUILDOPT_HTTP2
+
 /* Define if doing a development build */
 #undef BUILDOPT_IS_DEVEL_BUILD
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for libostree 2017.14.
+# Generated by GNU Autoconf 2.69 for libostree 2017.15.
 #
 # Report bugs to <walters@verbum.org>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='libostree'
 PACKAGE_TARNAME='libostree'
-PACKAGE_VERSION='2017.14'
+PACKAGE_VERSION='2017.15'
-PACKAGE_STRING='libostree 2017.14'
+PACKAGE_STRING='libostree 2017.15'
 PACKAGE_BUGREPORT='walters@verbum.org'
 PACKAGE_URL=''
 
@@ -925,6 +925,7 @@ enable_otmpfile
 enable_wrpseudo_compat
 enable_glibtest
 with_curl
+enable_http2
 with_soup
 enable_libsoup_client_certs
 enable_trivial_httpd_cmdline
@@ -1540,7 +1541,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures libostree 2017.14 to adapt to many kinds of systems.
+\`configure' configures libostree 2017.15 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1610,7 +1611,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of libostree 2017.14:";;
+     short | recursive ) echo "Configuration of libostree 2017.15:";;
    esac
   cat <<\_ACEOF
 
@@ -1642,6 +1643,7 @@ Optional Features:
                           Disable use syscall() and filesystem calls to for
                           compatibility with wrpseudo [default=no]
   --disable-glibtest      do not try to compile and run a test GLIB program
+  --disable-http2         Disable use of http2 (default: no)
   --enable-libsoup-client-certs
                           Require availability of new enough libsoup TLS
                           client cert API (default: auto)
@@ -1851,7 +1853,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-libostree configure 2017.14
+libostree configure 2017.15
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2323,7 +2325,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by libostree $as_me 2017.14, which was
+It was created by libostree $as_me 2017.15, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3191,7 +3193,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='libostree'
- VERSION='2017.14'
+ VERSION='2017.15'
 
 
 # Some tools Automake needs.
@@ -5925,9 +5927,9 @@ test -n "$YACC" || YACC="yacc"
 
 YEAR_VERSION=2017
 
-RELEASE_VERSION=14
+RELEASE_VERSION=15
 
-PACKAGE_VERSION=2017.14
+PACKAGE_VERSION=2017.15
 
 
 if echo "$CFLAGS" | grep -q -E -e '-Werror($| )'; then :
@@ -5954,6 +5956,7 @@ else
   -Werror=incompatible-pointer-types \
   -Werror=misleading-indentation \
   -Werror=missing-include-dirs -Werror=aggregate-return \
+  -Wstrict-aliasing=2 \
   -Werror=unused-result \
 ; do
 
@@ -14982,6 +14985,24 @@ else
 fi
 
 if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES libcurl"; fi
+# Check whether --enable-http2 was given.
+if test "${enable_http2+set}" = set; then :
+  enableval=$enable_http2;
+else
+  enable_http2=yes
+fi
+
+if test x$enable_http2 != xno ; then :
+
+
+$as_echo "#define BUILDOPT_HTTP2 1" >>confdefs.h
+
+
+else
+
+  OSTREE_FEATURES="$OSTREE_FEATURES no-http2"
+
+fi
 
 SOUP_DEPENDENCY="libsoup-2.4 >= 2.39.1"
 
@@ -18359,7 +18380,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by libostree $as_me 2017.14, which was
+This file was extended by libostree $as_me 2017.15, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -18425,7 +18446,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-libostree config.status 2017.14
+libostree config.status 2017.15
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
@@ -20116,6 +20137,7 @@ fi
 
 echo "
     libostree $VERSION ($release_build_type)
+    features: $OSTREE_FEATURES
     ===============
 
 

configure.ac

@@ -4,7 +4,7 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check
 dnl in test-symbols.sh, and also set is_release_build=yes below.  Then make
 dnl another post-release commit to bump the version, and set is_release_build=no.
 m4_define([year_version], [2017])
-m4_define([release_version], [14])
+m4_define([release_version], [15])
 m4_define([package_version], [year_version.release_version])
 AC_INIT([libostree], [package_version], [walters@verbum.org])
 is_release_build=yes
@@ -48,6 +48,7 @@ CC_CHECK_FLAGS_APPEND([WARN_CFLAGS], [CFLAGS], [\
   -Werror=incompatible-pointer-types \
   -Werror=misleading-indentation \
   -Werror=missing-include-dirs -Werror=aggregate-return \
+  -Wstrict-aliasing=2 \
   -Werror=unused-result \
 ])])
 AC_SUBST(WARN_CFLAGS)
@@ -135,6 +136,15 @@ AS_IF([test x$with_curl != xno ], [
 ], [with_soup_default=check])
 AM_CONDITIONAL(USE_CURL, test x$with_curl != xno)
 if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES libcurl"; fi
+AC_ARG_ENABLE(http2,
+AS_HELP_STRING([--disable-http2],
+  [Disable use of http2 (default: no)]),,
+  [enable_http2=yes])
+AS_IF([test x$enable_http2 != xno ], [
+  AC_DEFINE([BUILDOPT_HTTP2], 1, [Define if we enable http2])
+], [
+  OSTREE_FEATURES="$OSTREE_FEATURES no-http2"
+])
 
 dnl When bumping the libsoup-2.4 dependency, remember to bump
 dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in
@@ -556,6 +566,7 @@ AC_OUTPUT
 
 echo "
     libostree $VERSION ($release_build_type)
+    features: $OSTREE_FEATURES
     ===============
 
 

libglnx/glnx-console.c

@@ -29,9 +29,22 @@
 #include <errno.h>
 #include <sys/ioctl.h>
 
-static char *current_text = NULL;
+/* For people with widescreen monitors and maximized terminals, it looks pretty
-static gint current_percent = -1;
+ * bad to have an enormous progress bar. For much the same reason as web pages
+ * tend to have a maximum width;
+ * https://ux.stackexchange.com/questions/48982/suggest-good-max-width-for-fluid-width-design
+ */
+#define MAX_PROGRESSBAR_COLUMNS 20
+
+/* Max updates output per second.  On a tty there's no point to rendering
+ * extremely fast; and for a non-tty we're probably in a Jenkins job
+ * or whatever and having percentages spam multiple lines there is annoying.
+ */
+#define MAX_TTY_UPDATE_HZ (5)
+#define MAX_NONTTY_UPDATE_HZ (1)
+
 static gboolean locked;
+static guint64 last_update_ms; /* monotonic time in millis we last updated */
 
 static gboolean
 stdout_is_tty (void)
@@ -147,8 +160,6 @@ glnx_console_lock (GLnxConsoleRef *console)
 
   locked = console->locked = TRUE;
 
-  current_percent = 0;
-
   if (console->is_tty)
     {
       if (g_once_init_enter (&sigwinch_initialized))
@@ -181,6 +192,26 @@ static void
 text_percent_internal (const char *text,
                        int percentage)
 {
+  /* Check whether we're trying to render too fast; unless percentage is 100, in
+   * which case we assume this is the last call, so we always render it.
+   */
+  const guint64 current_ms = g_get_monotonic_time () / 1000;
+  if (percentage != 100)
+    {
+      const guint64 diff_ms = current_ms - last_update_ms;
+      if (stdout_is_tty ())
+        {
+          if (diff_ms < (1000/MAX_TTY_UPDATE_HZ))
+            return;
+        }
+      else
+        {
+          if (diff_ms < (1000/MAX_NONTTY_UPDATE_HZ))
+            return;
+        }
+    }
+  last_update_ms = current_ms;
+
   static const char equals[] = "====================";
   const guint n_equals = sizeof (equals) - 1;
   static const char spaces[] = "                    ";
@@ -193,10 +224,6 @@ text_percent_internal (const char *text,
 
   const guint input_textlen = text ? strlen (text) : 0;
 
-  if (percentage == current_percent
-      && g_strcmp0 (text, current_text) == 0)
-    return;
-
   if (!stdout_is_tty ())
     {
       if (text)
@@ -232,7 +259,7 @@ text_percent_internal (const char *text,
   else
     {
       const guint textlen = MIN (input_textlen, ncolumns - bar_min);
-      const guint barlen = ncolumns - (textlen + 1);;
+      const guint barlen = MIN (MAX_PROGRESSBAR_COLUMNS, ncolumns - (textlen + 1));
 
       if (textlen > 0)
         {
@@ -280,6 +307,32 @@ glnx_console_progress_text_percent (const char *text,
   text_percent_internal (text, percentage);
 }
 
+/**
+ * glnx_console_progress_n_items:
+ * @text: Show this text before the progress bar
+ * @current: An integer for how many items have been processed
+ * @total: An integer for how many items there are total
+ *
+ * On a tty, print to the console @text followed by [@current/@total],
+ * then an ASCII art progress bar, like glnx_console_progress_text_percent().
+ *
+ * You must have called glnx_console_lock() before invoking this
+ * function.
+ */
+void
+glnx_console_progress_n_items (const char     *text,
+                               guint           current,
+                               guint           total)
+{
+  g_return_if_fail (current <= total);
+  g_return_if_fail (total > 0);
+
+  g_autofree char *newtext = g_strdup_printf ("%s (%u/%u)", text, current, total);
+  /* Special case current == total to ensure we end at 100% */
+  int percentage = (current == total) ? 100 : (((double)current) / total * 100);
+  glnx_console_progress_text_percent (newtext, percentage);
+}
+
 void
 glnx_console_text (const char *text)
 {
@@ -299,9 +352,6 @@ glnx_console_unlock (GLnxConsoleRef *console)
   g_return_if_fail (locked);
   g_return_if_fail (console->locked);
 
-  current_percent = -1;
-  g_clear_pointer (&current_text, g_free);
-
   if (console->is_tty)
     fputc ('\n', stdout);
       

libglnx/glnx-console.h

@@ -38,6 +38,10 @@ void	 glnx_console_text (const char     *text);
 void	 glnx_console_progress_text_percent (const char     *text,
                                            guint           percentage);
 
+void	 glnx_console_progress_n_items (const char     *text,
+                                      guint           current,
+                                      guint           total);
+
 void	 glnx_console_unlock (GLnxConsoleRef *ref);
 
 guint    glnx_console_lines (void);

libglnx/glnx-errors.c

@@ -23,6 +23,32 @@
 #include <glnx-backport-autocleanups.h>
 #include <glnx-errors.h>
 
+/* Set @error with G_IO_ERROR/G_IO_ERROR_FAILED.
+ *
+ * This function returns %FALSE so it can be used conveniently in a single
+ * statement:
+ *
+ * ```
+ *   if (strcmp (foo, "somevalue") != 0)
+ *     return glnx_throw (error, "key must be somevalue, not '%s'", foo);
+ * ```
+ */
+gboolean
+glnx_throw (GError    **error,
+            const char *fmt,
+            ...)
+{
+  if (error == NULL)
+    return FALSE;
+
+  va_list args;
+  va_start (args, fmt);
+  GError *new = g_error_new_valist (G_IO_ERROR, G_IO_ERROR_FAILED, fmt, args);
+  va_end (args);
+  g_propagate_error (error, g_steal_pointer (&new));
+  return FALSE;
+}
+
 void
 glnx_real_set_prefix_error_va (GError     *error,
                                const char *format,
@@ -39,6 +65,30 @@ glnx_real_set_prefix_error_va (GError     *error,
   error->message = g_string_free (g_steal_pointer (&buf), FALSE);
 }
 
+/* Prepend to @error's message by `$prefix: ` where `$prefix` is computed via
+ * printf @fmt. Returns %FALSE so it can be used conveniently in a single
+ * statement:
+ *
+ * ```
+ *   if (!function_that_fails (s, error))
+ *     return glnx_throw_prefix (error, "while handling '%s'", s);
+ * ```
+ * */
+gboolean
+glnx_prefix_error (GError    **error,
+                   const char *fmt,
+                   ...)
+{
+  if (error == NULL)
+    return FALSE;
+
+  va_list args;
+  va_start (args, fmt);
+  glnx_real_set_prefix_error_va (*error, fmt, args);
+  va_end (args);
+  return FALSE;
+}
+
 void
 glnx_real_set_prefix_error_from_errno_va (GError     **error,
                                           gint         errsv,
@@ -54,3 +104,28 @@ glnx_real_set_prefix_error_from_errno_va (GError     **error,
                        g_strerror (errsv));
   glnx_real_set_prefix_error_va (*error, format, args);
 }
+
+/* Set @error using the value of `$prefix: g_strerror (errno)` where `$prefix`
+ * is computed via printf @fmt.
+ *
+ * This function returns %FALSE so it can be used conveniently in a single
+ * statement:
+ *
+ * ```
+ *   return glnx_throw_errno_prefix (error, "unlinking %s", pathname);
+ * ```
+ */
+gboolean
+glnx_throw_errno_prefix (GError    **error,
+                         const char *fmt,
+                         ...)
+{
+  int errsv = errno;
+  va_list args;
+  va_start (args, fmt);
+  glnx_real_set_prefix_error_from_errno_va (error, errsv, fmt, args);
+  va_end (args);
+  /* See comment in glnx_throw_errno() about preserving errno */
+  errno = errsv;
+  return FALSE;
+}

libglnx/glnx-errors.h

@@ -25,29 +25,7 @@
 
 G_BEGIN_DECLS
 
-/* Set @error with G_IO_ERROR/G_IO_ERROR_FAILED.
+gboolean glnx_throw (GError **error, const char *fmt, ...) G_GNUC_PRINTF (2,3);
- *
- * This function returns %FALSE so it can be used conveniently in a single
- * statement:
- *
- * ```
- *   if (strcmp (foo, "somevalue") != 0)
- *     return glnx_throw (error, "key must be somevalue, not '%s'", foo);
- * ```
- */
-static inline gboolean G_GNUC_PRINTF (2,3)
-glnx_throw (GError **error, const char *fmt, ...)
-{
-  if (error == NULL)
-    return FALSE;
-
-  va_list args;
-  va_start (args, fmt);
-  GError *new = g_error_new_valist (G_IO_ERROR, G_IO_ERROR_FAILED, fmt, args);
-  va_end (args);
-  g_propagate_error (error, g_steal_pointer (&new));
-  return FALSE;
-}
 
 /* Like `glnx_throw ()`, but returns %NULL. */
 #define glnx_null_throw(error, args...) \
@@ -58,27 +36,7 @@ void glnx_real_set_prefix_error_va (GError     *error,
                                     const char *format,
                                     va_list     args) G_GNUC_PRINTF (2,0);
 
-/* Prepend to @error's message by `$prefix: ` where `$prefix` is computed via
+gboolean glnx_prefix_error (GError **error, const char *fmt, ...) G_GNUC_PRINTF (2,3);
- * printf @fmt. Returns %FALSE so it can be used conveniently in a single
- * statement:
- *
- * ```
- *   if (!function_that_fails (s, error))
- *     return glnx_throw_prefix (error, "while handling '%s'", s);
- * ```
- * */
-static inline gboolean G_GNUC_PRINTF (2,3)
-glnx_prefix_error (GError **error, const char *fmt, ...)
-{
-  if (error == NULL)
-    return FALSE;
-
-  va_list args;
-  va_start (args, fmt);
-  glnx_real_set_prefix_error_va (*error, fmt, args);
-  va_end (args);
-  return FALSE;
-}
 
 /* Like `glnx_prefix_error ()`, but returns %NULL. */
 #define glnx_prefix_error_null(error, args...) \
@@ -155,28 +113,7 @@ void glnx_real_set_prefix_error_from_errno_va (GError     **error,
                                                const char  *format,
                                                va_list      args) G_GNUC_PRINTF (3,0);
 
-/* Set @error using the value of `$prefix: g_strerror (errno)` where `$prefix`
+gboolean glnx_throw_errno_prefix (GError **error, const char *fmt, ...) G_GNUC_PRINTF (2,3);
- * is computed via printf @fmt.
- *
- * This function returns %FALSE so it can be used conveniently in a single
- * statement:
- *
- * ```
- *   return glnx_throw_errno_prefix (error, "unlinking %s", pathname);
- * ```
- */
-static inline gboolean G_GNUC_PRINTF (2,3)
-glnx_throw_errno_prefix (GError **error, const char *fmt, ...)
-{
-  int errsv = errno;
-  va_list args;
-  va_start (args, fmt);
-  glnx_real_set_prefix_error_from_errno_va (error, errsv, fmt, args);
-  va_end (args);
-  /* See comment above about preserving errno */
-  errno = errsv;
-  return FALSE;
-}
 
 /* Like glnx_throw_errno_prefix(), but yields a NULL pointer. */
 #define glnx_null_throw_errno_prefix(error, args...) \

libglnx/glnx-fdio.h

@@ -29,12 +29,8 @@
 #include <string.h>
 #include <stdio.h>
 #include <sys/xattr.h>
-/* From systemd/src/shared/util.h */
+// For dirname(), and previously basename()
-/* When we include libgen.h because we need dirname() we immediately
- * undefine basename() since libgen.h defines it as a macro to the XDG
- * version which is really broken. */
 #include <libgen.h>
-#undef basename
 
 #include <glnx-macros.h>
 #include <glnx-errors.h>
@@ -47,7 +43,12 @@ G_BEGIN_DECLS
 static inline
 const char *glnx_basename (const char *path)
 {
-  return (basename) (path);
+  gchar *base = strrchr (path, G_DIR_SEPARATOR);
+
+  if (base)
+    return base + 1;
+
+  return path;
 }
 
 /* Utilities for standard FILE* */

man/ostree-fsck.xml

@@ -85,6 +85,27 @@ Boston, MA 02111-1307, USA.
                    Add tombstone commit for referenced but missing commits.
                 </para></listitem>
             </varlistentry>
+
+            <varlistentry>
+                <term><option>--verify-bindings</option></term>
+                <listitem><para>
+                  Verify that the commits pointed to by each ref have that
+                  ref in the binding set.  You should usually add this
+                  option; it only defaults to off for backwards compatibility.
+                </para></listitem>
+            </varlistentry>
+
+            <varlistentry>
+                <term><option>--verify-back-refs</option></term>
+                <listitem><para>
+                  Verify that all the refs listed in a commit’s ref-bindings
+                  point to that commit. This cannot be used in repositories
+                  where the target of refs is changed over time as new commits
+                  are added, but can be used in repositories which are
+                  regenerated from scratch for each commit.
+                  Implies <literal>--verify-bindings</literal> as well.
+                </para></listitem>
+            </varlistentry>
         </variablelist>
     </refsect1>
 

man/ostree.repo-config.xml

@@ -222,6 +222,14 @@ Boston, MA 02111-1307, USA.
         <listitem><para>Path to file containing trusted anchors instead of the system CA database.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><varname>http2</varname></term>
+        <listitem><para>A boolean value, defaults to true.  By
+        default, libostree will use HTTP2; setting this to <literal>false</literal>
+        will disable it.  May be useful to work around broken servers.
+        </para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><varname>unconfigured-state</varname></term>
         <listitem><para>If set, pulls from this remote will fail with the configured text.  This is intended for OS vendors which have a subscription process to access content.</para></listitem>

src/libostree/libostree-devel.sym

@@ -18,9 +18,8 @@
 ***/
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
-
+LIBOSTREE_2017.16 {
-LIBOSTREE_2017.15 {
+} LIBOSTREE_2017.15;
-} LIBOSTREE_2017.14;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the last number.  This is just a copy/paste

src/libostree/libostree-experimental.sym

@@ -94,4 +94,8 @@ LIBOSTREE_2017.14_EXPERIMENTAL {
 global:
   ostree_remote_get_type;
   ostree_remote_get_url;
+  ostree_repo_auto_lock_cleanup;
+  ostree_repo_auto_lock_push;
+  ostree_repo_lock_pop;
+  ostree_repo_lock_push;
 } LIBOSTREE_2017.13_EXPERIMENTAL;

src/libostree/libostree-released.sym

@@ -445,6 +445,12 @@ global:
 LIBOSTREE_2017.14 {
 } LIBOSTREE_2017.13;
 
+LIBOSTREE_2017.15 {
+  ostree_repo_fsck_object;
+  ostree_repo_mark_commit_partial;
+  ostree_break_hardlink;
+} LIBOSTREE_2017.14;
+
 /* NOTE: Only add more content here in release commits!  See the
  * comments at the top of this file.
  */

src/libostree/ostree-autocleanups.h

@@ -59,6 +59,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSysrootUpgrader, g_object_unref)
 G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (OstreeRepoCommitTraverseIter, ostree_repo_commit_traverse_iter_clear)
 
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeCollectionRef, ostree_collection_ref_free)
 G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeCollectionRefv, ostree_collection_ref_freev, NULL)
 G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRemote, ostree_remote_unref)

src/libostree/ostree-cmdprivate.c

@@ -22,6 +22,7 @@
 #include "ostree-cmdprivate.h"
 #include "ostree-repo-private.h"
 #include "ostree-core-private.h"
+#include "ostree-repo-pull-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-sysroot.h"
 #include "ostree-bootloader-grub2.h"
@@ -48,7 +49,8 @@ ostree_cmd__private__ (void)
     impl_ostree_generate_grub2_config,
     _ostree_repo_static_delta_dump,
     _ostree_repo_static_delta_query_exists,
-    _ostree_repo_static_delta_delete
+    _ostree_repo_static_delta_delete,
+    _ostree_repo_verify_bindings
   };
 
   return &table;

src/libostree/ostree-cmdprivate.h

@@ -31,6 +31,7 @@ typedef struct {
   gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
   gboolean (* ostree_static_delta_query_exists) (OstreeRepo *repo, const char *delta_id, gboolean *out_exists, GCancellable *cancellable, GError **error);
   gboolean (* ostree_static_delta_delete) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
+  gboolean (* ostree_repo_verify_bindings) (const char *collection_id, const char *ref_name, GVariant *commit, GError **error);
 } OstreeCmdPrivateVTable;
 
 /* Note this not really "public", we just export the symbol, but not the header */

src/libostree/ostree-core.c

@@ -749,6 +749,107 @@ ostree_content_file_parse (gboolean                compressed,
                                        cancellable, error);
 }
 
+static gboolean
+break_symhardlink (int                dfd,
+                   const char        *path,
+                   struct stat       *stbuf,
+                   GLnxFileCopyFlags  copyflags,
+                   GCancellable      *cancellable,
+                   GError           **error)
+{
+  guint count;
+  gboolean copy_success = FALSE;
+  char *path_tmp = glnx_strjoina (path, ".XXXXXX");
+
+  for (count = 0; count < 100; count++)
+    {
+      g_autoptr(GError) tmp_error = NULL;
+
+      glnx_gen_temp_name (path_tmp);
+
+      if (!glnx_file_copy_at (dfd, path, stbuf, dfd, path_tmp, copyflags,
+                              cancellable, &tmp_error))
+        {
+          if (g_error_matches (tmp_error, G_IO_ERROR, G_IO_ERROR_EXISTS))
+            continue;
+          g_propagate_error (error, g_steal_pointer (&tmp_error));
+          return FALSE;
+        }
+
+      copy_success = TRUE;
+      break;
+    }
+
+  if (!copy_success)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_EXISTS,
+                   "Exceeded limit of %u file creation attempts", count);
+      return FALSE;
+    }
+
+  if (!glnx_renameat (dfd, path_tmp, dfd, path, error))
+    return FALSE;
+
+  return TRUE;
+}
+
+/**
+ * ostree_break_hardlink:
+ * @dfd: Directory fd
+ * @path: Path relative to @dfd
+ * @skip_xattrs: Do not copy extended attributes
+ * @error: error
+ *
+ * In many cases using libostree, a program may need to "break"
+ * hardlinks by performing a copy.  For example, in order to
+ * logically append to a file.
+ *
+ * This function performs full copying, including e.g. extended
+ * attributes and permissions of both regular files and symbolic links.
+ *
+ * If the file is not hardlinked, this function does nothing and
+ * returns successfully.
+ *
+ * This function does not perform synchronization via `fsync()` or
+ * `fdatasync()`; the idea is this will commonly be done as part
+ * of an `ostree_repo_commit_transaction()`, which itself takes
+ * care of synchronization.
+ *
+ * Since: 2017.15
+ */
+gboolean ostree_break_hardlink (int               dfd,
+                                const char       *path,
+                                gboolean          skip_xattrs,
+                                GCancellable     *cancellable,
+                                GError          **error)
+{
+  struct stat stbuf;
+
+  if (!glnx_fstatat (dfd, path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+
+  if (stbuf.st_nlink <= 1)
+    return TRUE;  /* Note early return */
+
+  const GLnxFileCopyFlags copyflags = skip_xattrs ? GLNX_FILE_COPY_NOXATTRS : 0;
+
+  if (S_ISREG (stbuf.st_mode))
+    /* Note it's now completely safe to copy a file to itself,
+     * as glnx_file_copy_at() is documented to do an O_TMPFILE + rename()
+     * with GLNX_FILE_COPY_OVERWRITE.
+     */
+    return glnx_file_copy_at (dfd, path, &stbuf, dfd, path,
+                              copyflags | GLNX_FILE_COPY_OVERWRITE,
+                              cancellable, error);
+  else if (S_ISLNK (stbuf.st_mode))
+    return break_symhardlink (dfd, path, &stbuf, copyflags,
+                              cancellable, error);
+  else
+    return glnx_throw (error, "Unsupported type for entry '%s'", path);
+
+  return TRUE;
+}
+
 /**
  * ostree_checksum_file_from_input:
  * @file_info: File information

src/libostree/ostree-core.h

@@ -438,6 +438,13 @@ gboolean ostree_checksum_file (GFile             *f,
                                GCancellable      *cancellable,
                                GError           **error);
 
+_OSTREE_PUBLIC
+gboolean ostree_break_hardlink (int               dfd,
+                                const char       *path,
+                                gboolean          skip_xattrs,
+                                GCancellable     *cancellable,
+                                GError          **error);
+
 /**
  * OstreeChecksumFlags:
  *

src/libostree/ostree-fetcher-curl.c

@@ -771,6 +771,9 @@ initiate_next_curl_request (FetcherRequest *req,
    * there are numerous HTTP/2 fixes since the original version in
    * libcurl 7.43.0.
    */
+#ifdef BUILDOPT_HTTP2
+  if (!(self->config_flags & OSTREE_FETCHER_FLAGS_DISABLE_HTTP2))
+    {
 #if CURL_AT_LEAST_VERSION(7, 51, 0)
       curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 #endif
@@ -778,6 +781,8 @@ initiate_next_curl_request (FetcherRequest *req,
 #if (CURLPIPE_MULTIPLEX > 0)
       /* wait for pipe connection to confirm */
       curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L);
+#endif
+    }
 #endif
   curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
   if (g_getenv ("OSTREE_DEBUG_HTTP"))

src/libostree/ostree-fetcher.h

@@ -50,7 +50,8 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcher, g_object_unref)
 typedef enum {
   OSTREE_FETCHER_FLAGS_NONE = 0,
   OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0),
-  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1)
+  OSTREE_FETCHER_FLAGS_TRANSFER_GZIP = (1 << 1),
+  OSTREE_FETCHER_FLAGS_DISABLE_HTTP2 = (1 << 2),
 } OstreeFetcherConfigFlags;
 
 typedef enum {

src/libostree/ostree-repo-commit.c

@@ -597,11 +597,13 @@ static gboolean
 write_content_object (OstreeRepo         *self,
                       const char         *expected_checksum,
                       GInputStream       *input,
-                      guint64             file_object_length,
+                      GFileInfo          *file_info,
+                      GVariant           *xattrs,
                       guchar            **out_csum,
                       GCancellable       *cancellable,
                       GError            **error)
 {
+  GLNX_AUTO_PREFIX_ERROR ("Writing content object", error);
   g_return_val_if_fail (expected_checksum || out_csum, FALSE);
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
@@ -609,18 +611,30 @@ write_content_object (OstreeRepo         *self,
 
   OstreeRepoMode repo_mode = ostree_repo_get_mode (self);
 
+  GInputStream *file_input; /* Unowned alias */
+  g_autoptr(GInputStream) file_input_owned = NULL; /* We need a temporary for bare-user symlinks */
   glnx_unref_object OtChecksumInstream *checksum_input = NULL;
   if (out_csum)
-    checksum_input = ot_checksum_instream_new (input, G_CHECKSUM_SHA256);
+    {
-
+      /* Previously we checksummed the input verbatim; now
-  g_autoptr(GInputStream) file_input = NULL;
+       * ostree_repo_write_content() parses without checksumming, then we
-  g_autoptr(GVariant) xattrs = NULL;
+       * re-synthesize a header here. The data should be identical; if somehow
-  g_autoptr(GFileInfo) file_info = NULL;
+       * it's not that's not a serious problem because we're still computing a
-  if (!ostree_content_stream_parse (FALSE, checksum_input ? (GInputStream*)checksum_input : input,
+       * checksum over the data we actually use.
-                                    file_object_length, FALSE,
+       */
-                                    &file_input, &file_info, &xattrs,
+      g_autoptr(GBytes) header = _ostree_file_header_new (file_info, xattrs);
-                                    cancellable, error))
+      size_t len;
-    return FALSE;
+      const guint8 *buf = g_bytes_get_data (header, &len);
+      /* Give a null input if there's no content */
+      g_autoptr(GInputStream) null_input = NULL;
+      if (!input)
+        null_input = input = g_memory_input_stream_new_from_data ("", 0, NULL);
+      checksum_input = ot_checksum_instream_new_with_start (input, G_CHECKSUM_SHA256,
+                                                            buf, len);
+      file_input = (GInputStream*)checksum_input;
+    }
+  else
+    file_input = input;
 
   gboolean phys_object_is_symlink = FALSE;
   const GFileType object_file_type = g_file_info_get_file_type (file_info);
@@ -644,10 +658,8 @@ write_content_object (OstreeRepo         *self,
       const char *target_str = g_file_info_get_symlink_target (file_info);
       g_autoptr(GBytes) target = g_bytes_new (target_str, strlen (target_str) + 1);
 
-      if (file_input != NULL)
-        g_object_unref (file_input);
       /* Include the terminating zero so we can e.g. mmap this file */
-      file_input = g_memory_input_stream_new_from_bytes (target);
+      file_input = file_input_owned = g_memory_input_stream_new_from_bytes (target);
       size = g_bytes_get_size (target);
     }
   else if (!phys_object_is_symlink)
@@ -658,19 +670,19 @@ write_content_object (OstreeRepo         *self,
   /* Free space check; only applies during transactions */
   if (self->min_free_space_percent > 0 && self->in_transaction)
     {
-      g_mutex_lock (&self->txn_stats_lock);
+      g_mutex_lock (&self->txn_lock);
-      g_assert_cmpint (self->txn_blocksize, >, 0);
+      g_assert_cmpint (self->txn.blocksize, >, 0);
-      const fsblkcnt_t object_blocks = (size / self->txn_blocksize) + 1;
+      const fsblkcnt_t object_blocks = (size / self->txn.blocksize) + 1;
-      if (object_blocks > self->max_txn_blocks)
+      if (object_blocks > self->txn.max_blocks)
         {
-          g_mutex_unlock (&self->txn_stats_lock);
+          g_mutex_unlock (&self->txn_lock);
-          g_autofree char *formatted_required = g_format_size ((guint64)object_blocks * self->txn_blocksize);
+          g_autofree char *formatted_required = g_format_size ((guint64)object_blocks * self->txn.blocksize);
           return glnx_throw (error, "min-free-space-percent '%u%%' would be exceeded, %s more required",
                              self->min_free_space_percent, formatted_required);
         }
       /* This is the main bit that needs mutex protection */
-      self->max_txn_blocks -= object_blocks;
+      self->txn.max_blocks -= object_blocks;
-      g_mutex_unlock (&self->txn_stats_lock);
+      g_mutex_unlock (&self->txn_lock);
     }
 
   /* For regular files, we create them with default mode, and only
@@ -777,9 +789,9 @@ write_content_object (OstreeRepo         *self,
   /* If we already have it, just update the stats. */
   if (have_obj)
     {
-      g_mutex_lock (&self->txn_stats_lock);
+      g_mutex_lock (&self->txn_lock);
-      self->txn_stats.content_objects_total++;
+      self->txn.stats.content_objects_total++;
-      g_mutex_unlock (&self->txn_stats_lock);
+      g_mutex_unlock (&self->txn_lock);
       if (out_csum)
         *out_csum = ostree_checksum_to_bytes (actual_checksum);
       /* Note early return */
@@ -844,16 +856,15 @@ write_content_object (OstreeRepo         *self,
                                         uid, gid, mode,
                                         xattrs,
                                         cancellable, error))
-        return glnx_prefix_error (error, "Writing object %s.%s", actual_checksum,
+        return FALSE;
-                                  ostree_object_type_to_string (OSTREE_OBJECT_TYPE_FILE));
     }
 
   /* Update statistics */
-  g_mutex_lock (&self->txn_stats_lock);
+  g_mutex_lock (&self->txn_lock);
-  self->txn_stats.content_objects_written++;
+  self->txn.stats.content_objects_written++;
-  self->txn_stats.content_bytes_written += file_object_length;
+  self->txn.stats.content_bytes_written += g_file_info_get_size (file_info);
-  self->txn_stats.content_objects_total++;
+  self->txn.stats.content_objects_total++;
-  g_mutex_unlock (&self->txn_stats_lock);
+  g_mutex_unlock (&self->txn_lock);
 
   if (out_csum)
     {
@@ -886,6 +897,8 @@ adopt_and_commit_regfile (OstreeRepo   *self,
                           GCancellable *cancellable,
                           GError      **error)
 {
+  GLNX_AUTO_PREFIX_ERROR ("Commit regfile (adopt)", error);
+
   g_assert (G_IN_SET (self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER_ONLY));
   g_autoptr(GBytes) header = _ostree_file_header_new (finfo, xattrs);
 
@@ -981,6 +994,8 @@ write_metadata_object (OstreeRepo         *self,
                        GCancellable       *cancellable,
                        GError            **error)
 {
+  GLNX_AUTO_PREFIX_ERROR ("Writing metadata object", error);
+
   g_return_val_if_fail (expected_checksum || out_csum, FALSE);
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
@@ -1018,9 +1033,9 @@ write_metadata_object (OstreeRepo         *self,
        */
       if (have_obj)
         {
-          g_mutex_lock (&self->txn_stats_lock);
+          g_mutex_lock (&self->txn_lock);
-          self->txn_stats.metadata_objects_total++;
+          self->txn.stats.metadata_objects_total++;
-          g_mutex_unlock (&self->txn_stats_lock);
+          g_mutex_unlock (&self->txn_lock);
 
           if (out_csum)
             *out_csum = ostree_checksum_to_bytes (actual_checksum);
@@ -1090,10 +1105,10 @@ write_metadata_object (OstreeRepo         *self,
     }
 
   /* Update the stats, note we both wrote one and add to total */
-  g_mutex_lock (&self->txn_stats_lock);
+  g_mutex_lock (&self->txn_lock);
-  self->txn_stats.metadata_objects_written++;
+  self->txn.stats.metadata_objects_written++;
-  self->txn_stats.metadata_objects_total++;
+  self->txn.stats.metadata_objects_total++;
-  g_mutex_unlock (&self->txn_stats_lock);
+  g_mutex_unlock (&self->txn_lock);
 
   if (out_csum)
     *out_csum = ostree_checksum_to_bytes (actual_checksum);
@@ -1259,6 +1274,8 @@ devino_cache_lookup (OstreeRepo           *self,
  * existing ostree objects, then this will speed up considerably, so call it
  * before you call ostree_write_directory_to_mtree() or similar.  However,
  * ostree_repo_devino_cache_new() is better as it avoids scanning all objects.
+ *
+ * Multithreading: This function is *not* MT safe.
  */
 gboolean
 ostree_repo_scan_hardlinks (OstreeRepo    *self,
@@ -1287,8 +1304,19 @@ ostree_repo_scan_hardlinks (OstreeRepo    *self,
  * ostree_repo_commit_transaction(), or abort the transaction with
  * ostree_repo_abort_transaction().
  *
- * Currently, transactions are not atomic, and aborting a transaction
+ * Currently, transactions may result in partial commits or data in the target
- * will not erase any data you  write during the transaction.
+ * repository if interrupted during ostree_repo_commit_transaction(), and
+ * further writing refs is also not currently atomic.
+ *
+ * There can be at most one transaction active on a repo at a time per instance
+ * of `OstreeRepo`; however, it is safe to have multiple threads writing objects
+ * on a single `OstreeRepo` instance as long as their lifetime is bounded by the
+ * transaction.
+ *
+ * Multithreading: This function is *not* MT safe; only one transaction can be
+ * active at a time.
+ *
+ * This function takes a shared lock on the @self repository.
  */
 gboolean
 ostree_repo_prepare_transaction (OstreeRepo     *self,
@@ -1299,7 +1327,12 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
 
   g_return_val_if_fail (self->in_transaction == FALSE, FALSE);
 
-  memset (&self->txn_stats, 0, sizeof (OstreeRepoTransactionStats));
+  memset (&self->txn.stats, 0, sizeof (OstreeRepoTransactionStats));
+
+  self->txn_locked = ostree_repo_lock_push (self, OSTREE_REPO_LOCK_SHARED,
+                                            cancellable, error);
+  if (!self->txn_locked)
+    return FALSE;
 
   self->in_transaction = TRUE;
   if (self->min_free_space_percent > 0)
@@ -1307,23 +1340,23 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
       struct statvfs stvfsbuf;
       if (TEMP_FAILURE_RETRY (fstatvfs (self->repo_dir_fd, &stvfsbuf)) < 0)
         return glnx_throw_errno_prefix (error, "fstatvfs");
-      g_mutex_lock (&self->txn_stats_lock);
+      g_mutex_lock (&self->txn_lock);
-      self->txn_blocksize = stvfsbuf.f_bsize;
+      self->txn.blocksize = stvfsbuf.f_bsize;
       /* Convert fragment to blocks to compute the total */
       guint64 total_blocks = (stvfsbuf.f_frsize * stvfsbuf.f_blocks) / stvfsbuf.f_bsize;
       /* Use the appropriate free block count if we're unprivileged */
       guint64 bfree = (getuid () != 0 ? stvfsbuf.f_bavail : stvfsbuf.f_bfree);
       guint64 reserved_blocks = ((double)total_blocks) * (self->min_free_space_percent/100.0);
       if (bfree > reserved_blocks)
-        self->max_txn_blocks = bfree - reserved_blocks;
+        self->txn.max_blocks = bfree - reserved_blocks;
       else
         {
-          g_mutex_unlock (&self->txn_stats_lock);
+          g_mutex_unlock (&self->txn_lock);
-          g_autofree char *formatted_free = g_format_size (bfree * self->txn_blocksize);
+          g_autofree char *formatted_free = g_format_size (bfree * self->txn.blocksize);
           return glnx_throw (error, "min-free-space-percent '%u%%' would be exceeded, %s available",
                              self->min_free_space_percent, formatted_free);
         }
-      g_mutex_unlock (&self->txn_stats_lock);
+      g_mutex_unlock (&self->txn_lock);
     }
 
   gboolean ret_transaction_resume = FALSE;
@@ -1547,15 +1580,57 @@ cleanup_tmpdir (OstreeRepo        *self,
 static void
 ensure_txn_refs (OstreeRepo *self)
 {
-  if (self->txn_refs == NULL)
+  if (self->txn.refs == NULL)
-    self->txn_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
+    self->txn.refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-  if (self->txn_collection_refs == NULL)
+  if (self->txn.collection_refs == NULL)
-    self->txn_collection_refs = g_hash_table_new_full (ostree_collection_ref_hash,
+    self->txn.collection_refs = g_hash_table_new_full (ostree_collection_ref_hash,
                                                        ostree_collection_ref_equal,
                                                        (GDestroyNotify) ostree_collection_ref_free,
                                                        g_free);
 }
 
+/**
+ * ostree_repo_mark_commit_partial:
+ * @self: Repo
+ * @checksum: Commit SHA-256
+ * @is_partial: Whether or not this commit is partial
+ * @error: Error
+ *
+ * Commits in "partial" state do not have all their child objects written. This
+ * occurs in various situations, such as during a pull, but also if a "subpath"
+ * pull is used, as well as "commit only" pulls.
+ *
+ * This function is used by ostree_repo_pull_with_options(); you
+ * should use this if you are implementing a different type of transport.
+ *
+ * Since: 2017.15
+ */
+gboolean
+ostree_repo_mark_commit_partial (OstreeRepo     *self,
+                                 const char     *checksum,
+                                 gboolean        is_partial,
+                                 GError        **error)
+{
+  g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
+  if (is_partial)
+    {
+      glnx_autofd int fd = openat (self->repo_dir_fd, commitpartial_path,
+                                   O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644);
+      if (fd == -1)
+        {
+          if (errno != EEXIST)
+            return glnx_throw_errno_prefix (error, "open(%s)", commitpartial_path);
+        }
+    }
+  else
+    {
+      if (!ot_ensure_unlinked_at (self->repo_dir_fd, commitpartial_path, 0))
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
 /**
  * ostree_repo_transaction_set_refspec:
  * @self: An #OstreeRepo
@@ -1565,6 +1640,8 @@ ensure_txn_refs (OstreeRepo *self)
  * Like ostree_repo_transaction_set_ref(), but takes concatenated
  * @refspec format as input instead of separate remote and name
  * arguments.
+ *
+ * Multithreading: Since v2017.15 this function is MT safe.
  */
 void
 ostree_repo_transaction_set_refspec (OstreeRepo *self,
@@ -1573,9 +1650,10 @@ ostree_repo_transaction_set_refspec (OstreeRepo *self,
 {
   g_return_if_fail (self->in_transaction == TRUE);
 
+  g_mutex_lock (&self->txn_lock);
   ensure_txn_refs (self);
-
+  g_hash_table_replace (self->txn.refs, g_strdup (refspec), g_strdup (checksum));
-  g_hash_table_replace (self->txn_refs, g_strdup (refspec), g_strdup (checksum));
+  g_mutex_unlock (&self->txn_lock);
 }
 
 /**
@@ -1592,10 +1670,20 @@ ostree_repo_transaction_set_refspec (OstreeRepo *self,
  * Otherwise, if @checksum is %NULL, then record that the ref should
  * be deleted.
  *
- * The change will not be written out immediately, but when the transaction
+ * The change will be written when the transaction is completed with
- * is completed with ostree_repo_commit_transaction(). If the transaction
+ * ostree_repo_commit_transaction(); that function takes care of writing all of
- * is instead aborted with ostree_repo_abort_transaction(), no changes will
+ * the objects (such as the commit referred to by @checksum) before updating the
- * be made to the repository.
+ * refs. If the transaction is instead aborted with
+ * ostree_repo_abort_transaction(), no changes to the ref will be made to the
+ * repository.
+ *
+ * Note however that currently writing *multiple* refs is not truly atomic; if
+ * the process or system is terminated during
+ * ostree_repo_commit_transaction(), it is possible that just some of the refs
+ * will have been updated. Your application should take care to handle this
+ * case.
+ *
+ * Multithreading: Since v2017.15 this function is MT safe.
  */
 void
 ostree_repo_transaction_set_ref (OstreeRepo *self,
@@ -1603,18 +1691,18 @@ ostree_repo_transaction_set_ref (OstreeRepo *self,
                                  const char *ref,
                                  const char *checksum)
 {
-  char *refspec;
-
   g_return_if_fail (self->in_transaction == TRUE);
 
-  ensure_txn_refs (self);
+  char *refspec;
-
   if (remote)
     refspec = g_strdup_printf ("%s:%s", remote, ref);
   else
     refspec = g_strdup (ref);
 
-  g_hash_table_replace (self->txn_refs, refspec, g_strdup (checksum));
+  g_mutex_lock (&self->txn_lock);
+  ensure_txn_refs (self);
+  g_hash_table_replace (self->txn.refs, refspec, g_strdup (checksum));
+  g_mutex_unlock (&self->txn_lock);
 }
 
 /**
@@ -1634,6 +1722,8 @@ ostree_repo_transaction_set_ref (OstreeRepo *self,
  * is instead aborted with ostree_repo_abort_transaction(), no changes will
  * be made to the repository.
  *
+ * Multithreading: Since v2017.15 this function is MT safe.
+ *
  * Since: 2017.8
  */
 void
@@ -1646,10 +1736,11 @@ ostree_repo_transaction_set_collection_ref (OstreeRepo                *self,
   g_return_if_fail (ref != NULL);
   g_return_if_fail (checksum == NULL || ostree_validate_checksum_string (checksum, NULL));
 
+  g_mutex_lock (&self->txn_lock);
   ensure_txn_refs (self);
-
+  g_hash_table_replace (self->txn.collection_refs,
-  g_hash_table_replace (self->txn_collection_refs,
                         ostree_collection_ref_dup (ref), g_strdup (checksum));
+  g_mutex_unlock (&self->txn_lock);
 }
 
 /**
@@ -1664,6 +1755,8 @@ ostree_repo_transaction_set_collection_ref (OstreeRepo                *self,
  * This is like ostree_repo_transaction_set_ref(), except it may be
  * invoked outside of a transaction.  This is presently safe for the
  * case where we're creating or overwriting an existing ref.
+ *
+ * Multithreading: This function is MT safe.
  */
 gboolean
 ostree_repo_set_ref_immediate (OstreeRepo *self,
@@ -1745,6 +1838,12 @@ ostree_repo_set_collection_ref_immediate (OstreeRepo                 *self,
  * Complete the transaction. Any refs set with
  * ostree_repo_transaction_set_ref() or
  * ostree_repo_transaction_set_refspec() will be written out.
+ *
+ * Note that if multiple threads are performing writes, all such threads must
+ * have terminated before this function is invoked.
+ *
+ * Multithreading: This function is *not* MT safe; only one transaction can be
+ * active at a time.
  */
 gboolean
 ostree_repo_commit_transaction (OstreeRepo                  *self,
@@ -1782,23 +1881,30 @@ ostree_repo_commit_transaction (OstreeRepo                  *self,
   if (self->loose_object_devino_hash)
     g_hash_table_remove_all (self->loose_object_devino_hash);
 
-  if (self->txn_refs)
+  if (self->txn.refs)
-    if (!_ostree_repo_update_refs (self, self->txn_refs, cancellable, error))
+    if (!_ostree_repo_update_refs (self, self->txn.refs, cancellable, error))
       return FALSE;
-  g_clear_pointer (&self->txn_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.refs, g_hash_table_destroy);
 
-  if (self->txn_collection_refs)
+  if (self->txn.collection_refs)
-    if (!_ostree_repo_update_collection_refs (self, self->txn_collection_refs, cancellable, error))
+    if (!_ostree_repo_update_collection_refs (self, self->txn.collection_refs, cancellable, error))
       return FALSE;
-  g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.collection_refs, g_hash_table_destroy);
 
   self->in_transaction = FALSE;
 
   if (!ot_ensure_unlinked_at (self->repo_dir_fd, "transaction", 0))
     return FALSE;
 
+  if (self->txn_locked)
+    {
+      if (!ostree_repo_lock_pop (self, cancellable, error))
+        return FALSE;
+      self->txn_locked = FALSE;
+    }
+
   if (out_stats)
-    *out_stats = self->txn_stats;
+    *out_stats = self->txn.stats;
 
   return TRUE;
 }
@@ -1829,14 +1935,21 @@ ostree_repo_abort_transaction (OstreeRepo     *self,
   if (self->loose_object_devino_hash)
     g_hash_table_remove_all (self->loose_object_devino_hash);
 
-  g_clear_pointer (&self->txn_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.refs, g_hash_table_destroy);
-  g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.collection_refs, g_hash_table_destroy);
 
   glnx_tmpdir_unset (&self->commit_stagedir);
   glnx_release_lock_file (&self->commit_stagedir_lock);
 
   self->in_transaction = FALSE;
 
+  if (self->txn_locked)
+    {
+      if (!ostree_repo_lock_pop (self, cancellable, error))
+        return FALSE;
+      self->txn_locked = FALSE;
+    }
+
   return TRUE;
 }
 
@@ -2186,8 +2299,17 @@ ostree_repo_write_content (OstreeRepo       *self,
         }
     }
 
+  /* Parse the stream */
+  g_autoptr(GInputStream) file_input = NULL;
+  g_autoptr(GVariant) xattrs = NULL;
+  g_autoptr(GFileInfo) file_info = NULL;
+  if (!ostree_content_stream_parse (FALSE, object_input, length, FALSE,
+                                    &file_input, &file_info, &xattrs,
+                                    cancellable, error))
+    return FALSE;
+
   return write_content_object (self, expected_checksum,
-                               object_input, length, out_csum,
+                               file_input, file_info, xattrs, out_csum,
                                cancellable, error);
 }
 
@@ -2773,12 +2895,112 @@ typedef enum {
   WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT = 1,
 } WriteDirContentFlags;
 
-/* Given either a dir_enum or a dfd_iter, writes the directory entry to the mtree. For
+/* Given either a dir_enum or a dfd_iter, writes the directory entry (which is
- * subdirs, we go back through either write_dfd_iter_to_mtree_internal (dfd_iter case) or
+ * itself a directory) to the mtree. For subdirs, we go back through either
- * write_directory_to_mtree_internal (dir_enum case) which will do the actual dirmeta +
+ * write_dfd_iter_to_mtree_internal (dfd_iter case) or
- * dirent iteration. */
+ * write_directory_to_mtree_internal (dir_enum case) which will do the actual
+ * dirmeta + dirent iteration. */
 static gboolean
-write_directory_content_to_mtree_internal (OstreeRepo                  *self,
+write_dir_entry_to_mtree_internal (OstreeRepo                  *self,
+                                   OstreeRepoFile              *repo_dir,
+                                   GFileEnumerator             *dir_enum,
+                                   GLnxDirFdIterator           *dfd_iter,
+                                   WriteDirContentFlags         writeflags,
+                                   GFileInfo                   *child_info,
+                                   OstreeMutableTree           *mtree,
+                                   OstreeRepoCommitModifier    *modifier,
+                                   GPtrArray                   *path,
+                                   GCancellable                *cancellable,
+                                   GError                     **error)
+{
+  g_assert (dir_enum != NULL || dfd_iter != NULL);
+  g_assert (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY);
+
+  const char *name = g_file_info_get_name (child_info);
+
+  /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even
+   * more complexity in this function, and it'd mostly only be useful when
+   * operating on local filesystems anyways.
+   */
+  const gboolean delete_after_commit = dfd_iter && modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+
+  /* Build the full path which we need for callbacks */
+  g_ptr_array_add (path, (char*)name);
+  g_autofree char *child_relpath = ptrarray_path_join (path);
+
+  /* Call the filter */
+  g_autoptr(GFileInfo) modified_info = NULL;
+  OstreeRepoCommitFilterResult filter_result =
+    _ostree_repo_commit_modifier_apply (self, modifier, child_relpath, child_info, &modified_info);
+
+  if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW)
+    {
+      g_ptr_array_remove_index (path, path->len - 1);
+      if (delete_after_commit)
+        {
+          g_assert (dfd_iter);
+          if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
+            return FALSE;
+        }
+      /* Note: early return */
+      return TRUE;
+    }
+
+  g_autoptr(GFile) child = NULL;
+  if (dir_enum != NULL)
+    child = g_file_enumerator_get_child (dir_enum, child_info);
+
+  g_autoptr(OstreeMutableTree) child_mtree = NULL;
+  if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
+    return FALSE;
+
+  /* Finally, recurse on the dir */
+  if (dir_enum != NULL)
+    {
+      if (!write_directory_to_mtree_internal (self, child, child_mtree,
+                                              modifier, path,
+                                              cancellable, error))
+        return FALSE;
+    }
+  else if (repo_dir)
+    {
+      g_assert (dir_enum != NULL);
+      g_debug ("Adding: %s", gs_file_get_path_cached (child));
+      if (!ostree_mutable_tree_replace_file (mtree, name,
+                                             ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
+                                             error))
+        return FALSE;
+    }
+  else
+    {
+      g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, };
+
+      if (!glnx_dirfd_iterator_init_at (dfd_iter->fd, name, FALSE, &child_dfd_iter, error))
+        return FALSE;
+
+      if (!write_dfd_iter_to_mtree_internal (self, &child_dfd_iter, child_mtree,
+                                             modifier, path,
+                                             cancellable, error))
+        return FALSE;
+
+      if (delete_after_commit)
+        {
+          if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
+            return FALSE;
+        }
+    }
+
+  g_ptr_array_remove_index (path, path->len - 1);
+
+  return TRUE;
+}
+
+/* Given either a dir_enum or a dfd_iter, writes a non-dir (regfile/symlink) to
+ * the mtree.
+ */
+static gboolean
+write_content_to_mtree_internal (OstreeRepo                  *self,
                                            OstreeRepoFile              *repo_dir,
                                            GFileEnumerator             *dir_enum,
                                            GLnxDirFdIterator           *dfd_iter,
@@ -2811,7 +3033,7 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
 
   /* See if we have a devino hit; this is used below in a few places. */
   const char *loose_checksum = NULL;
-  if (dfd_iter != NULL && (file_type != G_FILE_TYPE_DIRECTORY))
+  if (dfd_iter != NULL)
     {
       guint32 dev = g_file_info_get_attribute_uint32 (child_info, "unix::device");
       guint64 inode = g_file_info_get_attribute_uint64 (child_info, "unix::inode");
@@ -2844,12 +3066,13 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
    * there.
    */
   g_autoptr(GVariant) source_xattrs = NULL;
+  g_autoptr(GFileInfo) source_child_info = NULL;
   if (loose_checksum && self->mode == OSTREE_REPO_MODE_BARE_USER)
     {
-      child_info = NULL;
+      if (!ostree_repo_load_file (self, loose_checksum, NULL, &source_child_info, &source_xattrs,
-      if (!ostree_repo_load_file (self, loose_checksum, NULL, &child_info, &source_xattrs,
                                   cancellable, error))
         return FALSE;
+      child_info = source_child_info;
     }
 
   /* Call the filter */
@@ -2873,7 +3096,6 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
 
   switch (file_type)
     {
-    case G_FILE_TYPE_DIRECTORY:
     case G_FILE_TYPE_SYMBOLIC_LINK:
     case G_FILE_TYPE_REGULAR:
       break;
@@ -2885,49 +3107,7 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
   if (dir_enum != NULL)
     child = g_file_enumerator_get_child (dir_enum, child_info);
 
-  if (file_type == G_FILE_TYPE_DIRECTORY)
+  /* Our filters have passed, etc.; now we prepare to write the content object */
-    {
-      g_autoptr(OstreeMutableTree) child_mtree = NULL;
-      if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
-        return FALSE;
-
-      if (dir_enum != NULL)
-        {
-          if (!write_directory_to_mtree_internal (self, child, child_mtree,
-                                                  modifier, path,
-                                                  cancellable, error))
-            return FALSE;
-        }
-      else
-        {
-          g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, };
-
-          if (!glnx_dirfd_iterator_init_at (dfd_iter->fd, name, FALSE, &child_dfd_iter, error))
-            return FALSE;
-
-          if (!write_dfd_iter_to_mtree_internal (self, &child_dfd_iter, child_mtree,
-                                                 modifier, path,
-                                                 cancellable, error))
-            return FALSE;
-
-          if (delete_after_commit)
-            {
-              if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
-                return FALSE;
-            }
-        }
-    }
-  else if (repo_dir)
-    {
-      g_assert (dir_enum != NULL);
-      g_debug ("Adding: %s", gs_file_get_path_cached (child));
-      if (!ostree_mutable_tree_replace_file (mtree, name,
-                                             ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
-                                             error))
-        return FALSE;
-    }
-  else
-    {
   glnx_autofd int file_input_fd = -1;
 
   /* Open the file now, since it's better for reading xattrs
@@ -3034,15 +3214,8 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
             }
         }
 
-          g_autoptr(GInputStream) file_object_input = NULL;
-          guint64 file_obj_length;
-          if (!ostree_raw_file_to_content_stream (file_input,
-                                                  modified_info, xattrs,
-                                                  &file_object_input, &file_obj_length,
-                                                  cancellable, error))
-            return FALSE;
       g_autofree guchar *child_file_csum = NULL;
-          if (!ostree_repo_write_content (self, NULL, file_object_input, file_obj_length,
+      if (!write_content_object (self, NULL, file_input, modified_info, xattrs,
                                  &child_file_csum, cancellable, error))
         return FALSE;
 
@@ -3061,7 +3234,6 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
       if (!glnx_unlinkat (dfd_iter->fd, name, 0, error))
         return FALSE;
     }
-    }
 
   g_ptr_array_remove_index (path, path->len - 1);
 
@@ -3069,7 +3241,7 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
 }
 
 /* Handles the dirmeta for the given GFile dir and then calls
- * write_directory_content_to_mtree_internal() for each directory entry. */
+ * write_{dir_entry,content}_to_mtree_internal() for each directory entry. */
 static gboolean
 write_directory_to_mtree_internal (OstreeRepo                  *self,
                                    GFile                       *dir,
@@ -3154,7 +3326,18 @@ write_directory_to_mtree_internal (OstreeRepo                  *self,
           if (child_info == NULL)
             break;
 
-          if (!write_directory_content_to_mtree_internal (self, repo_dir, dir_enum, NULL,
+          if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
+            {
+              if (!write_dir_entry_to_mtree_internal (self, repo_dir, dir_enum, NULL,
+                                                      WRITE_DIR_CONTENT_FLAGS_NONE,
+                                                      child_info,
+                                                      mtree, modifier, path,
+                                                      cancellable, error))
+                return FALSE;
+            }
+          else
+            {
+              if (!write_content_to_mtree_internal (self, repo_dir, dir_enum, NULL,
                                                     WRITE_DIR_CONTENT_FLAGS_NONE,
                                                     child_info,
                                                     mtree, modifier, path,
@@ -3162,12 +3345,13 @@ write_directory_to_mtree_internal (OstreeRepo                  *self,
                 return FALSE;
             }
         }
+    }
 
   return TRUE;
 }
 
 /* Handles the dirmeta for the dir described by src_dfd_iter and then calls
- * write_directory_content_to_mtree_internal() for each directory entry. */
+ * write_{dir_entry,content}_to_mtree_internal() for each directory entry. */
 static gboolean
 write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
                                   GLnxDirFdIterator           *src_dfd_iter,
@@ -3244,6 +3428,18 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
       g_autoptr(GFileInfo) child_info = _ostree_stbuf_to_gfileinfo (&stbuf);
       g_file_info_set_name (child_info, dent->d_name);
 
+      if (S_ISDIR (stbuf.st_mode))
+        {
+          if (!write_dir_entry_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
+                                                  flags, child_info,
+                                                  mtree, modifier, path,
+                                                  cancellable, error))
+            return FALSE;
+
+          /* We handled the dir, move onto the next */
+          continue;
+        }
+
       if (S_ISREG (stbuf.st_mode))
         ;
       else if (S_ISLNK (stbuf.st_mode))
@@ -3252,15 +3448,14 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
                                          child_info, cancellable, error))
             return FALSE;
         }
-      else if (S_ISDIR (stbuf.st_mode))
-        ;
       else
         {
           return glnx_throw (error, "Not a regular file or symlink: %s",
                              dent->d_name);
         }
 
-      if (!write_directory_content_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
+      /* Write a content object, we handled directories above */
+      if (!write_content_to_mtree_internal (self, NULL, NULL, src_dfd_iter,
                                             flags, child_info,
                                             mtree, modifier, path,
                                             cancellable, error))

src/libostree/ostree-repo-file.c

@@ -508,7 +508,7 @@ ostree_repo_file_get_parent (GFile *file)
 {
   OstreeRepoFile *self = OSTREE_REPO_FILE (file);
 
-  return g_object_ref (self->parent);
+  return (GFile*)g_object_ref (self->parent);
 }
 
 static GFile *
@@ -621,7 +621,7 @@ ostree_repo_file_resolve_relative_path (GFile      *file,
       g_assert (*relative_path == '/');
 
       if (strcmp (relative_path, "/") == 0)
-        return g_object_ref (ostree_repo_file_get_root (self)); 
+        return (GFile*)g_object_ref (ostree_repo_file_get_root (self));
 
       if (self->parent)
         return ostree_repo_file_resolve_relative_path ((GFile*)ostree_repo_file_get_root (self),

src/libostree/ostree-repo-private.h

@@ -82,6 +82,15 @@ typedef enum {
   OSTREE_REPO_SYSROOT_KIND_IS_SYSROOT_OSTREE, /* We match /ostree/repo */
 } OstreeRepoSysrootKind;
 
+typedef struct {
+  GHashTable *refs;  /* (element-type utf8 utf8) */
+  GHashTable *collection_refs;  /* (element-type OstreeCollectionRef utf8) */
+  OstreeRepoTransactionStats stats;
+  /* Implementation of min-free-space-percent */
+  gulong blocksize;
+  fsblkcnt_t max_blocks;
+} OstreeRepoTxn;
+
 /**
  * OstreeRepo:
  *
@@ -109,13 +118,9 @@ struct OstreeRepo {
   GWeakRef sysroot; /* Weak to avoid a circular ref; see also `is_system` */
   char *remotes_config_dir;
 
-  GHashTable *txn_refs;  /* (element-type utf8 utf8) */
+  GMutex txn_lock;
-  GHashTable *txn_collection_refs;  /* (element-type OstreeCollectionRef utf8) */
+  OstreeRepoTxn txn;
-  GMutex txn_stats_lock;
+  gboolean txn_locked;
-  OstreeRepoTransactionStats txn_stats;
-  /* Implementation of min-free-space-percent */
-  gulong txn_blocksize;
-  fsblkcnt_t max_txn_blocks;
 
   GMutex cache_lock;
   guint dirmeta_cache_refcount;
@@ -153,6 +158,7 @@ struct OstreeRepo {
   guint64 tmp_expiry_seconds;
   gchar *collection_id;
   gboolean add_remotes_config_dir; /* Add new remotes in remotes.d dir */
+  gint lock_timeout_seconds;
 
   OstreeRepo *parent_repo;
 };
@@ -432,6 +438,32 @@ _ostree_repo_get_remote_inherited (OstreeRepo  *self,
 
 #ifndef OSTREE_ENABLE_EXPERIMENTAL_API
 
+/* All the locking APIs below are duplicated in ostree-repo.h. Remove the ones
+ * here once it's no longer experimental.
+ */
+
+typedef enum {
+  OSTREE_REPO_LOCK_SHARED,
+  OSTREE_REPO_LOCK_EXCLUSIVE
+} OstreeRepoLockType;
+
+gboolean      ostree_repo_lock_push (OstreeRepo          *self,
+                                     OstreeRepoLockType   lock_type,
+                                     GCancellable        *cancellable,
+                                     GError             **error);
+gboolean      ostree_repo_lock_pop (OstreeRepo    *self,
+                                    GCancellable  *cancellable,
+                                    GError       **error);
+
+typedef OstreeRepo OstreeRepoAutoLock;
+
+OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo          *self,
+                                                 OstreeRepoLockType   lock_type,
+                                                 GCancellable        *cancellable,
+                                                 GError             **error);
+void          ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock);
+G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoAutoLock, ostree_repo_auto_lock_cleanup)
+
 const gchar * ostree_repo_get_collection_id (OstreeRepo   *self);
 gboolean      ostree_repo_set_collection_id (OstreeRepo   *self,
                                              const gchar  *collection_id,

src/libostree/ostree-repo-prune.c

@@ -23,6 +23,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-autocleanups.h"
 #include "otutil.h"
 
 typedef struct {
@@ -35,16 +36,6 @@ typedef struct {
   guint64 freed_bytes;
 } OtPruneData;
 
-static gboolean
-prune_commitpartial_file (OstreeRepo    *repo,
-                          const char    *checksum,
-                          GCancellable  *cancellable,
-                          GError       **error)
-{
-  g_autofree char *path = _ostree_get_commitpartial_path (checksum);
-  return ot_ensure_unlinked_at (repo->repo_dir_fd, path, error);
-}
-
 static gboolean
 maybe_prune_loose_object (OtPruneData        *data,
                           OstreeRepoPruneFlags    flags,
@@ -67,7 +58,7 @@ maybe_prune_loose_object (OtPruneData        *data,
 
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
             {
-              if (!prune_commitpartial_file (data->repo, checksum, cancellable, error))
+              if (!ostree_repo_mark_commit_partial (data->repo, checksum, FALSE, error))
                 return FALSE;
             }
 
@@ -160,12 +151,20 @@ _ostree_repo_prune_tmp (OstreeRepo *self,
  * Prune static deltas, if COMMIT is specified then delete static delta files only
  * targeting that commit; otherwise any static delta of non existing commits are
  * deleted.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit,
                                  GCancellable      *cancellable,
                                  GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GPtrArray) deltas = NULL;
   if (!ostree_repo_list_static_delta_names (self, &deltas,
                                             cancellable, error))
@@ -286,6 +285,8 @@ repo_prune_internal (OstreeRepo        *self,
  * Use the %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE to just determine
  * statistics on objects that would be deleted, without actually
  * deleting them.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune (OstreeRepo        *self,
@@ -297,6 +298,12 @@ ostree_repo_prune (OstreeRepo        *self,
                    GCancellable      *cancellable,
                    GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GHashTable) objects = NULL;
   gboolean refs_only = flags & OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY;
 
@@ -391,6 +398,8 @@ ostree_repo_prune (OstreeRepo        *self,
  *
  * The %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE flag may be specified to just determine
  * statistics on objects that would be deleted, without actually deleting them.
+ *
+ * This function takes an exclusive lock on the @self repository.
  */
 gboolean
 ostree_repo_prune_from_reachable (OstreeRepo        *self,
@@ -401,6 +410,12 @@ ostree_repo_prune_from_reachable (OstreeRepo        *self,
                                   GCancellable      *cancellable,
                                   GError           **error)
 {
+  g_autoptr(OstreeRepoAutoLock) lock =
+    ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable,
+                                error);
+  if (!lock)
+    return FALSE;
+
   g_autoptr(GHashTable) objects = NULL;
 
   if (!ostree_repo_list_objects (self, OSTREE_REPO_LIST_OBJECTS_ALL | OSTREE_REPO_LIST_OBJECTS_NO_PARENTS,

src/libostree/ostree-repo-pull-private.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright © 2017 Endless Mobile, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#include "ostree-core.h"
+
+G_BEGIN_DECLS
+
+gboolean
+_ostree_repo_verify_bindings (const char  *collection_id,
+                              const char  *ref_name,
+                              GVariant    *commit,
+                              GError     **error);
+
+G_END_DECLS

src/libostree/ostree-repo-pull.c

@@ -32,6 +32,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-repo-pull-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
 #include "ostree-fetcher-util.h"
@@ -557,21 +558,6 @@ fetch_uri_contents_utf8_sync (OstreeFetcher  *fetcher,
                                                 cancellable, error);
 }
 
-static gboolean
-write_commitpartial_for (OtPullData *pull_data,
-                         const char *checksum,
-                         GError **error)
-{
-  g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
-  glnx_autofd int fd = openat (pull_data->repo->repo_dir_fd, commitpartial_path, O_EXCL | O_CREAT | O_WRONLY | O_CLOEXEC | O_NOCTTY, 0644);
-  if (fd == -1)
-    {
-      if (errno != EEXIST)
-        return glnx_throw_errno_prefix (error, "open(%s)", commitpartial_path);
-    }
-  return TRUE;
-}
-
 static void
 enqueue_one_object_request (OtPullData                *pull_data,
                             const char                *checksum,
@@ -1266,7 +1252,7 @@ meta_fetch_on_complete (GObject           *object,
                                             pull_data->cancellable, error))
             goto out;
 
-          if (!write_commitpartial_for (pull_data, checksum, error))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
             goto out;
         }
 
@@ -1475,30 +1461,40 @@ get_remote_repo_collection_id (OtPullData *pull_data)
 }
 #endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
 
-/* Verify the ref and collection bindings.
+#endif  /* HAVE_LIBCURL_OR_LIBSOUP */
+
+/**
+ * _ostree_repo_verify_bindings:
+ * @collection_id: (nullable): Locally specified collection ID for the remote
+ *    the @commit was retrieved from, or %NULL if none is configured
+ * @ref_name: (nullable): Ref name the commit was retrieved using, or %NULL if
+ *    the commit was retrieved by checksum
+ * @commit: Commit data to check
+ * @error: Return location for a #GError, or %NULL
+ *
+ * Verify the ref and collection bindings.
  *
  * The ref binding is verified only if it exists. But if we have the
- * collection ID specified in the remote configuration then the ref
+ * collection ID specified in the remote configuration (@collection_id is
- * binding must exist, otherwise the verification will fail. Parts of
+ * non-%NULL) then the ref binding must exist, otherwise the verification will
- * the verification can be skipped by passing NULL to the requested_ref
+ * fail. Parts of the verification can be skipped by passing %NULL to the
- * parameter (in case we requested a checksum directly, without looking it up
+ * @ref_name parameter (in case we requested a checksum directly, without
- * from a ref).
+ * looking it up from a ref).
  *
  * The collection binding is verified only when we have collection ID
  * specified in the remote configuration. If it is specified, then the
  * binding must exist and must be equal to the remote repository
  * collection ID.
+ *
+ * Returns: %TRUE if bindings are correct, %FALSE otherwise
+ * Since: 2017.14
  */
-static gboolean
+gboolean
-verify_bindings (OtPullData                 *pull_data,
+_ostree_repo_verify_bindings (const char  *collection_id,
+                              const char  *ref_name,
                               GVariant    *commit,
-                 const OstreeCollectionRef  *requested_ref,
                               GError     **error)
 {
-  g_autofree char *remote_collection_id = NULL;
-#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
-  remote_collection_id = get_remote_repo_collection_id (pull_data);
-#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
   g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0);
   g_autofree const char **refs = NULL;
   if (!g_variant_lookup (metadata,
@@ -1510,17 +1506,17 @@ verify_bindings (OtPullData                 *pull_data,
        * we certainly will not verify the collection binding in the
        * commit.
        */
-      if (remote_collection_id == NULL)
+      if (collection_id == NULL)
         return TRUE;
 
       return glnx_throw (error,
-                         "expected commit metadata to have ref "
+                         "Expected commit metadata to have ref "
                          "binding information, found none");
     }
 
-  if (requested_ref != NULL)
+  if (ref_name != NULL)
     {
-      if (!g_strv_contains ((const char *const *) refs, requested_ref->ref_name))
+      if (!g_strv_contains ((const char *const *) refs, ref_name))
         {
           g_autoptr(GString) refs_dump = g_string_new (NULL);
           const char *refs_str;
@@ -1543,35 +1539,37 @@ verify_bindings (OtPullData                 *pull_data,
               refs_str = "no refs";
             }
 
-          return glnx_throw (error, "commit has no requested ref ‘%s’ "
+          return glnx_throw (error, "Commit has no requested ref ‘%s’ "
                              "in ref binding metadata (%s)",
-                             requested_ref->ref_name, refs_str);
+                             ref_name, refs_str);
         }
     }
 
-  if (remote_collection_id != NULL)
+  if (collection_id != NULL)
     {
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
-      const char *collection_id;
+      const char *collection_id_binding;
       if (!g_variant_lookup (metadata,
                              OSTREE_COMMIT_META_KEY_COLLECTION_BINDING,
                              "&s",
-                             &collection_id))
+                             &collection_id_binding))
         return glnx_throw (error,
-                           "expected commit metadata to have collection ID "
+                           "Expected commit metadata to have collection ID "
                            "binding information, found none");
-      if (!g_str_equal (collection_id, remote_collection_id))
+      if (!g_str_equal (collection_id_binding, collection_id))
         return glnx_throw (error,
-                           "commit has collection ID ‘%s’ in collection binding "
+                           "Commit has collection ID ‘%s’ in collection binding "
                            "metadata, while the remote it came from has "
                            "collection ID ‘%s’",
-                           collection_id, remote_collection_id);
+                           collection_id_binding, collection_id);
 #endif
     }
 
   return TRUE;
 }
 
+#ifdef HAVE_LIBCURL_OR_LIBSOUP
+
 /* Look at a commit object, and determine whether there are
  * more things to fetch.
  */
@@ -1626,7 +1624,13 @@ scan_commit_object (OtPullData                 *pull_data,
   /* If ref is non-NULL then the commit we fetched was requested through the
    * branch, otherwise we requested a commit checksum without specifying a branch.
    */
-  if (!verify_bindings (pull_data, commit, ref, error))
+  g_autofree char *remote_collection_id = NULL;
+#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+  remote_collection_id = get_remote_repo_collection_id (pull_data);
+#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
+  if (!_ostree_repo_verify_bindings (remote_collection_id,
+                                     (ref != NULL) ? ref->ref_name : NULL,
+                                     commit, error))
     return glnx_prefix_error (error, "Commit %s", checksum);
 
   if (pull_data->timestamp_check)
@@ -1802,7 +1806,7 @@ scan_one_metadata_object (OtPullData                 *pull_data,
       if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
         {
           /* mark as partial to ensure we scan the commit below */
-          if (!write_commitpartial_for (pull_data, checksum, error))
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
             return FALSE;
         }
 
@@ -1835,7 +1839,7 @@ scan_one_metadata_object (OtPullData                 *pull_data,
           if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
             {
               /* mark as partial to ensure we scan the commit below */
-              if (!write_commitpartial_for (pull_data, checksum, error))
+              if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error))
                 return FALSE;
             }
           if (!_ostree_repo_import_object (pull_data->repo, refd_repo,
@@ -2690,6 +2694,15 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
   if (gzip)
     fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP;
 
+  { gboolean http2 = TRUE;
+    if (!ostree_repo_get_remote_boolean_option (self, remote_name,
+                                                "http2", TRUE,
+                                                &http2, error))
+      goto out;
+    if (!http2)
+      fetcher_flags |= OSTREE_FETCHER_FLAGS_DISABLE_HTTP2;
+  }
+
   fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
 
   {
@@ -4312,15 +4325,13 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     {
       GLNX_HASH_TABLE_FOREACH_V (requested_refs_to_fetch, const char*, checksum)
         {
-          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (checksum);
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, FALSE, error))
-          if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
             goto out;
         }
 
       GLNX_HASH_TABLE_FOREACH_V (commits_to_fetch, const char*, commit)
         {
-          g_autofree char *commitpartial_path = _ostree_get_commitpartial_path (commit);
+          if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error))
-          if (!ot_ensure_unlinked_at (pull_data->repo->repo_dir_fd, commitpartial_path, 0))
             goto out;
         }
     }

src/libostree/ostree-repo-refs.c

@@ -726,13 +726,17 @@ _ostree_repo_list_refs_internal (OstreeRepo       *self,
  * @self: Repo
  * @refspec_prefix: (allow-none): Only list refs which match this prefix
  * @out_all_refs: (out) (element-type utf8 utf8) (transfer container):
- *    Mapping from ref to checksum
+ *    Mapping from refspec to checksum
  * @cancellable: Cancellable
  * @error: Error
  *
  * If @refspec_prefix is %NULL, list all local and remote refspecs,
  * with their current values in @out_all_refs.  Otherwise, only list
  * refspecs which have @refspec_prefix as a prefix.
+ *
+ * @out_all_refs will be returned as a mapping from refspecs (including the
+ * remote name) to checksums. If @refspec_prefix is non-%NULL, it will be
+ * removed as a prefix from the hash table keys.
  */
 gboolean
 ostree_repo_list_refs (OstreeRepo       *self,
@@ -752,16 +756,18 @@ ostree_repo_list_refs (OstreeRepo       *self,
  * @self: Repo
  * @refspec_prefix: (allow-none): Only list refs which match this prefix
  * @out_all_refs: (out) (element-type utf8 utf8) (transfer container):
- *    Mapping from ref to checksum
+ *    Mapping from refspec to checksum
  * @flags: Options controlling listing behavior
  * @cancellable: Cancellable
  * @error: Error
  *
  * If @refspec_prefix is %NULL, list all local and remote refspecs,
  * with their current values in @out_all_refs.  Otherwise, only list
- * refspecs which have @refspec_prefix as a prefix.  Differently from
+ * refspecs which have @refspec_prefix as a prefix.
- * ostree_repo_list_refs(), the prefix will not be removed from the ref
+ *
- * name.
+ * @out_all_refs will be returned as a mapping from refspecs (including the
+ * remote name) to checksums. Differently from ostree_repo_list_refs(), the
+ * @refspec_prefix will not be removed from the refspecs in the hash table.
  */
 gboolean
 ostree_repo_list_refs_ext (OstreeRepo                 *self,

src/libostree/ostree-repo-static-delta-private.h

@@ -128,11 +128,6 @@ _ostree_static_delta_part_open (GInputStream   *part_in,
                                 GCancellable *cancellable,
                                 GError      **error);
 
-gboolean _ostree_static_delta_dump (OstreeRepo     *repo,
-                                    const char *delta_id,
-                                    GCancellable   *cancellable,
-                                    GError        **error);
-
 typedef struct {
   guint n_ops_executed[OSTREE_STATIC_DELTA_N_OPS];
 } OstreeDeltaExecuteStats;

src/libostree/ostree-repo.c

@@ -46,6 +46,9 @@
 #include <sys/file.h>
 #include <sys/statvfs.h>
 
+#define REPO_LOCK_DISABLED (-2)
+#define REPO_LOCK_BLOCKING (-1)
+
 /* ABI Size checks for ostree-repo.h, only for LP64 systems;
  * https://en.wikipedia.org/wiki/64-bit_computing#64-bit_data_models
  *
@@ -156,6 +159,526 @@ G_DEFINE_TYPE (OstreeRepo, ostree_repo, G_TYPE_OBJECT)
 
 #define SYSCONF_REMOTES SHORTENED_SYSCONFDIR "/ostree/remotes.d"
 
+/* Repository locking
+ *
+ * To guard against objects being deleted (e.g., prune) while they're in
+ * use by another operation is accessing them (e.g., commit), the
+ * repository must be locked by concurrent writers.
+ *
+ * The locking is implemented by maintaining a thread local table of
+ * lock stacks per repository. This allows thread safe locking since
+ * each thread maintains its own lock stack. See the OstreeRepoLock type
+ * below.
+ *
+ * The actual locking is done using either open file descriptor locks or
+ * flock locks. This allows the locking to work with concurrent
+ * processes. The lock file is held on the ".lock" file within the
+ * repository.
+ *
+ * The intended usage is to take a shared lock when writing objects or
+ * reading objects in critical sections. Exclusive locks are taken when
+ * deleting objects.
+ *
+ * To allow fine grained locking within libostree, the lock is
+ * maintained as a stack. The core APIs then push or pop from the stack.
+ * When pushing or popping a lock state identical to the existing or
+ * next state, the stack is simply updated. Only when upgrading or
+ * downgrading the lock (changing to/from unlocked, pushing exclusive on
+ * shared or popping exclusive to shared) are actual locking operations
+ * performed.
+ */
+
+static void
+free_repo_lock_table (gpointer data)
+{
+  GHashTable *lock_table = data;
+
+  if (lock_table != NULL)
+    {
+      g_debug ("Free lock table");
+      g_hash_table_destroy (lock_table);
+    }
+}
+
+static GPrivate repo_lock_table = G_PRIVATE_INIT (free_repo_lock_table);
+
+typedef struct {
+  int fd;
+  GQueue stack;
+} OstreeRepoLock;
+
+typedef struct {
+  guint len;
+  int state;
+  const char *name;
+} OstreeRepoLockInfo;
+
+static void
+repo_lock_info (OstreeRepoLock *lock, OstreeRepoLockInfo *out_info)
+{
+  g_assert (lock != NULL);
+  g_assert (out_info != NULL);
+
+  OstreeRepoLockInfo info;
+  info.len = g_queue_get_length (&lock->stack);
+  if (info.len == 0)
+    {
+      info.state = LOCK_UN;
+      info.name = "unlocked";
+    }
+  else
+    {
+      info.state = GPOINTER_TO_INT (g_queue_peek_head (&lock->stack));
+      info.name = (info.state == LOCK_EX) ? "exclusive" : "shared";
+    }
+
+  *out_info = info;
+}
+
+static void
+free_repo_lock (gpointer data)
+{
+  OstreeRepoLock *lock = data;
+
+  if (lock != NULL)
+    {
+      OstreeRepoLockInfo info;
+      repo_lock_info (lock, &info);
+
+      g_debug ("Free lock: state=%s, depth=%u", info.name, info.len);
+      g_queue_clear (&lock->stack);
+      if (lock->fd >= 0)
+        {
+          g_debug ("Closing repo lock file");
+          (void) close (lock->fd);
+        }
+      g_free (lock);
+    }
+}
+
+/* Wrapper to handle flock vs OFD locking based on GLnxLockFile */
+static gboolean
+do_repo_lock (int fd,
+              int flags)
+{
+  int res;
+
+#ifdef F_OFD_SETLK
+  struct flock fl = {
+    .l_type = (flags & ~LOCK_NB) == LOCK_EX ? F_WRLCK : F_RDLCK,
+    .l_whence = SEEK_SET,
+    .l_start = 0,
+    .l_len = 0,
+  };
+
+  res = TEMP_FAILURE_RETRY (fcntl (fd, (flags & LOCK_NB) ? F_OFD_SETLK : F_OFD_SETLKW, &fl));
+#else
+  res = -1;
+  errno = EINVAL;
+#endif
+
+  /* Fallback to flock when OFD locks not available */
+  if (res < 0)
+    {
+      if (errno == EINVAL)
+        res = TEMP_FAILURE_RETRY (flock (fd, flags));
+      if (res < 0)
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+/* Wrapper to handle flock vs OFD unlocking based on GLnxLockFile */
+static gboolean
+do_repo_unlock (int fd,
+                int flags)
+{
+  int res;
+
+#ifdef F_OFD_SETLK
+  struct flock fl = {
+    .l_type = F_UNLCK,
+    .l_whence = SEEK_SET,
+    .l_start = 0,
+    .l_len = 0,
+  };
+
+  res = TEMP_FAILURE_RETRY (fcntl (fd, (flags & LOCK_NB) ? F_OFD_SETLK : F_OFD_SETLKW, &fl));
+#else
+  res = -1;
+  errno = EINVAL;
+#endif
+
+  /* Fallback to flock when OFD locks not available */
+  if (res < 0)
+    {
+      if (errno == EINVAL)
+        res = TEMP_FAILURE_RETRY (flock (fd, LOCK_UN | flags));
+      if (res < 0)
+        return FALSE;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+push_repo_lock (OstreeRepo          *self,
+                OstreeRepoLockType   lock_type,
+                gboolean             blocking,
+                GError             **error)
+{
+  int flags = (lock_type == OSTREE_REPO_LOCK_EXCLUSIVE) ? LOCK_EX : LOCK_SH;
+  if (!blocking)
+    flags |= LOCK_NB;
+
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  if (lock_table == NULL)
+    {
+      g_debug ("Creating repo lock table");
+      lock_table = g_hash_table_new_full (NULL, NULL, NULL,
+                                          (GDestroyNotify)free_repo_lock);
+      g_private_set (&repo_lock_table, lock_table);
+    }
+
+  OstreeRepoLock *lock = g_hash_table_lookup (lock_table, self);
+  if (lock == NULL)
+    {
+      lock = g_new0 (OstreeRepoLock, 1);
+      g_queue_init (&lock->stack);
+      g_debug ("Opening repo lock file");
+      lock->fd = TEMP_FAILURE_RETRY (openat (self->repo_dir_fd, ".lock",
+                                             O_CREAT | O_RDWR | O_CLOEXEC,
+                                             0600));
+      if (lock->fd < 0)
+        {
+          free_repo_lock (lock);
+          return glnx_throw_errno_prefix (error,
+                                          "Opening lock file %s/.lock failed",
+                                          gs_file_get_path_cached (self->repodir));
+        }
+      g_hash_table_insert (lock_table, self, lock);
+    }
+
+  OstreeRepoLockInfo info;
+  repo_lock_info (lock, &info);
+  g_debug ("Push lock: state=%s, depth=%u", info.name, info.len);
+
+  if (info.state == LOCK_EX)
+    {
+      g_debug ("Repo already locked exclusively, extending stack");
+      g_queue_push_head (&lock->stack, GINT_TO_POINTER (LOCK_EX));
+    }
+  else
+    {
+      int next_state = (flags & LOCK_EX) ? LOCK_EX : LOCK_SH;
+      const char *next_state_name = (flags & LOCK_EX) ? "exclusive" : "shared";
+
+      g_debug ("Locking repo %s", next_state_name);
+      if (!do_repo_lock (lock->fd, flags))
+        return glnx_throw_errno_prefix (error, "Locking repo %s failed",
+                                        next_state_name);
+
+      g_queue_push_head (&lock->stack, GINT_TO_POINTER (next_state));
+    }
+
+  return TRUE;
+}
+
+static gboolean
+pop_repo_lock (OstreeRepo  *self,
+               gboolean     blocking,
+               GError     **error)
+{
+  int flags = blocking ? 0 : LOCK_NB;
+
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  g_return_val_if_fail (lock_table != NULL, FALSE);
+
+  OstreeRepoLock *lock = g_hash_table_lookup (lock_table, self);
+  g_return_val_if_fail (lock != NULL, FALSE);
+  g_return_val_if_fail (lock->fd != -1, FALSE);
+
+  OstreeRepoLockInfo info;
+  repo_lock_info (lock, &info);
+  g_return_val_if_fail (info.len > 0, FALSE);
+
+  g_debug ("Pop lock: state=%s, depth=%u", info.name, info.len);
+  if (info.len > 1)
+    {
+      int next_state = GPOINTER_TO_INT (g_queue_peek_nth (&lock->stack, 1));
+
+      /* Drop back to the previous lock state if it differs */
+      if (next_state != info.state)
+        {
+          /* We should never drop from shared to exclusive */
+          g_return_val_if_fail (next_state == LOCK_SH, FALSE);
+          g_debug ("Returning lock state to shared");
+          if (!do_repo_lock (lock->fd, next_state | flags))
+            return glnx_throw_errno_prefix (error,
+                                            "Setting repo lock to shared failed");
+        }
+      else
+        g_debug ("Maintaining lock state as %s", info.name);
+    }
+  else
+    {
+      /* Lock stack will be empty, unlock */
+      g_debug ("Unlocking repo");
+      if (!do_repo_unlock (lock->fd, flags))
+        return glnx_throw_errno_prefix (error, "Unlocking repo failed");
+    }
+
+  g_queue_pop_head (&lock->stack);
+
+  return TRUE;
+}
+
+/**
+ * ostree_repo_lock_push:
+ * @self: a #OstreeRepo
+ * @lock_type: the type of lock to acquire
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Takes a lock on the repository and adds it to the lock stack. If @lock_type
+ * is %OSTREE_REPO_LOCK_SHARED, a shared lock is taken. If @lock_type is
+ * %OSTREE_REPO_LOCK_EXCLUSIVE, an exclusive lock is taken. The actual lock
+ * state is only changed when locking a previously unlocked repository or
+ * upgrading the lock from shared to exclusive. If the requested lock state is
+ * unchanged or would represent a downgrade (exclusive to shared), the lock
+ * state is not changed and the stack is simply updated.
+ *
+ * ostree_repo_lock_push() waits for the lock depending on the repository's
+ * lock-timeout configuration. When lock-timeout is -1, a blocking lock is
+ * attempted. Otherwise, the lock is taken non-blocking and
+ * ostree_repo_lock_push() will sleep synchronously up to lock-timeout seconds
+ * attempting to acquire the lock. If the lock cannot be acquired within the
+ * timeout, a %G_IO_ERROR_WOULD_BLOCK error is returned.
+ *
+ * If @self is not writable by the user, then no locking is attempted and
+ * %TRUE is returned.
+ *
+ * Returns: %TRUE on success, otherwise %FALSE with @error set
+ * Since: 2017.14
+ */
+gboolean
+ostree_repo_lock_push (OstreeRepo          *self,
+                       OstreeRepoLockType   lock_type,
+                       GCancellable        *cancellable,
+                       GError             **error)
+{
+  g_return_val_if_fail (self != NULL, FALSE);
+  g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
+  g_return_val_if_fail (self->inited, FALSE);
+  g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE);
+  g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
+
+  if (!self->writable)
+    return TRUE;
+
+  g_assert (self->lock_timeout_seconds >= REPO_LOCK_DISABLED);
+  if (self->lock_timeout_seconds == REPO_LOCK_DISABLED)
+    return TRUE; /* No locking */
+  else if (self->lock_timeout_seconds == REPO_LOCK_BLOCKING)
+    {
+      g_debug ("Pushing lock blocking");
+      return push_repo_lock (self, lock_type, TRUE, error);
+    }
+  else
+    {
+      /* Convert to unsigned to guard against negative values */
+      guint lock_timeout_seconds = self->lock_timeout_seconds;
+      guint waited = 0;
+      g_debug ("Pushing lock non-blocking with timeout %u",
+               lock_timeout_seconds);
+      for (;;)
+        {
+          if (g_cancellable_set_error_if_cancelled (cancellable, error))
+            return FALSE;
+
+          g_autoptr(GError) local_error = NULL;
+          if (push_repo_lock (self, lock_type, FALSE, &local_error))
+            return TRUE;
+
+          if (!g_error_matches (local_error, G_IO_ERROR,
+                                G_IO_ERROR_WOULD_BLOCK))
+            {
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          if (waited >= lock_timeout_seconds)
+            {
+              g_debug ("Push lock: Could not acquire lock within %u seconds",
+                       lock_timeout_seconds);
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          /* Sleep 1 second and try again */
+          if (waited % 60 == 0)
+            {
+              guint remaining = lock_timeout_seconds - waited;
+              g_debug ("Push lock: Waiting %u more second%s to acquire lock",
+                       remaining, (remaining == 1) ? "" : "s");
+            }
+          waited++;
+          sleep (1);
+        }
+    }
+}
+
+/**
+ * ostree_repo_lock_pop:
+ * @self: a #OstreeRepo
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Remove the current repository lock state from the lock stack. If the lock
+ * stack becomes empty, the repository is unlocked. Otherwise, the lock state
+ * only changes when transitioning from an exclusive lock back to a shared
+ * lock.
+ *
+ * ostree_repo_lock_pop() waits for the lock depending on the repository's
+ * lock-timeout configuration. When lock-timeout is -1, a blocking lock is
+ * attempted. Otherwise, the lock is removed non-blocking and
+ * ostree_repo_lock_pop() will sleep synchronously up to lock-timeout seconds
+ * attempting to remove the lock. If the lock cannot be removed within the
+ * timeout, a %G_IO_ERROR_WOULD_BLOCK error is returned.
+ *
+ * If @self is not writable by the user, then no unlocking is attempted and
+ * %TRUE is returned.
+ *
+ * Returns: %TRUE on success, otherwise %FALSE with @error set
+ * Since: 2017.14
+ */
+gboolean
+ostree_repo_lock_pop (OstreeRepo    *self,
+                      GCancellable  *cancellable,
+                      GError       **error)
+{
+  g_return_val_if_fail (self != NULL, FALSE);
+  g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE);
+  g_return_val_if_fail (self->inited, FALSE);
+  g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE);
+  g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
+
+  if (!self->writable)
+    return TRUE;
+
+  g_assert (self->lock_timeout_seconds >= REPO_LOCK_DISABLED);
+  if (self->lock_timeout_seconds == REPO_LOCK_DISABLED)
+    return TRUE;
+  else if (self->lock_timeout_seconds == REPO_LOCK_BLOCKING)
+    {
+      g_debug ("Popping lock blocking");
+      return pop_repo_lock (self, TRUE, error);
+    }
+  else
+    {
+      /* Convert to unsigned to guard against negative values */
+      guint lock_timeout_seconds = self->lock_timeout_seconds;
+      guint waited = 0;
+      g_debug ("Popping lock non-blocking with timeout %u",
+               lock_timeout_seconds);
+      for (;;)
+        {
+          if (g_cancellable_set_error_if_cancelled (cancellable, error))
+            return FALSE;
+
+          g_autoptr(GError) local_error = NULL;
+          if (pop_repo_lock (self, FALSE, &local_error))
+            return TRUE;
+
+          if (!g_error_matches (local_error, G_IO_ERROR,
+                                G_IO_ERROR_WOULD_BLOCK))
+            {
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          if (waited >= lock_timeout_seconds)
+            {
+              g_debug ("Pop lock: Could not remove lock within %u seconds",
+                       lock_timeout_seconds);
+              g_propagate_error (error, g_steal_pointer (&local_error));
+              return FALSE;
+            }
+
+          /* Sleep 1 second and try again */
+          if (waited % 60 == 0)
+            {
+              guint remaining = lock_timeout_seconds - waited;
+              g_debug ("Pop lock: Waiting %u more second%s to remove lock",
+                       remaining, (remaining == 1) ? "" : "s");
+            }
+          waited++;
+          sleep (1);
+        }
+    }
+}
+
+/**
+ * ostree_repo_auto_lock_push: (skip)
+ * @self: a #OstreeRepo
+ * @lock_type: the type of lock to acquire
+ * @cancellable: a #GCancellable
+ * @error: a #GError
+ *
+ * Like ostree_repo_lock_push(), but for usage with #OstreeRepoAutoLock.
+ * The intended usage is to declare the #OstreeRepoAutoLock with
+ * g_autoptr() so that ostree_repo_auto_lock_cleanup() is called when it
+ * goes out of scope. This will automatically pop the lock status off
+ * the stack if it was acquired successfully.
+ *
+ * |[<!-- language="C" -->
+ * g_autoptr(OstreeRepoAutoLock) lock = NULL;
+ * lock = ostree_repo_auto_lock_push (repo, lock_type, cancellable, error);
+ * if (!lock)
+ *   return FALSE;
+ * ]|
+ *
+ * Returns: @self on success, otherwise %NULL with @error set
+ * Since: 2017.14
+ */
+OstreeRepoAutoLock *
+ostree_repo_auto_lock_push (OstreeRepo          *self,
+                            OstreeRepoLockType   lock_type,
+                            GCancellable        *cancellable,
+                            GError             **error)
+{
+  if (!ostree_repo_lock_push (self, lock_type, cancellable, error))
+    return NULL;
+  return (OstreeRepoAutoLock *)self;
+}
+
+/**
+ * ostree_repo_auto_lock_cleanup: (skip)
+ * @lock: a #OstreeRepoAutoLock
+ *
+ * A cleanup handler for use with ostree_repo_auto_lock_push(). If @lock is
+ * not %NULL, ostree_repo_lock_pop() will be called on it. If
+ * ostree_repo_lock_pop() fails, a critical warning will be emitted.
+ *
+ * Since: 2017.14
+ */
+void
+ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock)
+{
+  OstreeRepo *repo = lock;
+  if (repo)
+    {
+      g_autoptr(GError) error = NULL;
+      int errsv = errno;
+
+      if (!ostree_repo_lock_pop (repo, NULL, &error))
+        g_critical ("Cleanup repo lock failed: %s", error->message);
+
+      errno = errsv;
+    }
+}
+
 static GFile *
 get_remotes_d_dir (OstreeRepo          *self,
                    GFile               *sysroot);
@@ -505,18 +1028,26 @@ ostree_repo_finalize (GObject *object)
     g_hash_table_destroy (self->updated_uncompressed_dirs);
   if (self->config)
     g_key_file_free (self->config);
-  g_clear_pointer (&self->txn_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.refs, g_hash_table_destroy);
-  g_clear_pointer (&self->txn_collection_refs, g_hash_table_destroy);
+  g_clear_pointer (&self->txn.collection_refs, g_hash_table_destroy);
   g_clear_error (&self->writable_error);
   g_clear_pointer (&self->object_sizes, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&self->dirmeta_cache, (GDestroyNotify) g_hash_table_unref);
   g_mutex_clear (&self->cache_lock);
-  g_mutex_clear (&self->txn_stats_lock);
+  g_mutex_clear (&self->txn_lock);
   g_free (self->collection_id);
 
   g_clear_pointer (&self->remotes, g_hash_table_destroy);
   g_mutex_clear (&self->remotes_lock);
 
+  GHashTable *lock_table = g_private_get (&repo_lock_table);
+  if (lock_table)
+    {
+      g_hash_table_remove (lock_table, self);
+      if (g_hash_table_size (lock_table) == 0)
+        g_private_replace (&repo_lock_table, NULL);
+    }
+
   G_OBJECT_CLASS (ostree_repo_parent_class)->finalize (object);
 }
 
@@ -672,7 +1203,7 @@ ostree_repo_init (OstreeRepo *self)
                                                  test_error_keys, G_N_ELEMENTS (test_error_keys));
 
   g_mutex_init (&self->cache_lock);
-  g_mutex_init (&self->txn_stats_lock);
+  g_mutex_init (&self->txn_lock);
 
   self->remotes = g_hash_table_new_full (g_str_hash, g_str_equal,
                                          (GDestroyNotify) NULL,
@@ -1266,7 +1797,7 @@ _ostree_repo_remote_list (OstreeRepo *self,
   g_mutex_unlock (&self->remotes_lock);
 
   if (self->parent_repo)
-    _ostree_repo_remote_list (self, out);
+    _ostree_repo_remote_list (self->parent_repo, out);
 }
 
 /**
@@ -1797,6 +2328,7 @@ repo_create_at_internal (int             dfd,
                          GCancellable   *cancellable,
                          GError        **error)
 {
+  GLNX_AUTO_PREFIX_ERROR ("Creating repo", error);
    struct stat stbuf;
   /* We do objects/ last - if it exists we do nothing and exit successfully */
   const char *state_dirs[] = { "tmp", "extensions", "state",
@@ -2208,6 +2740,27 @@ reload_core_config (OstreeRepo          *self,
     self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10);
   }
 
+  /* Disable locking by default for now */
+  { gboolean locking;
+    if (!ot_keyfile_get_boolean_with_default (self->config, "core", "locking",
+                                              FALSE, &locking, error))
+      return FALSE;
+    if (!locking)
+      {
+        self->lock_timeout_seconds = REPO_LOCK_DISABLED;
+      }
+    else
+      {
+        g_autofree char *lock_timeout_seconds = NULL;
+
+        if (!ot_keyfile_get_value_with_default (self->config, "core", "lock-timeout-secs", "30",
+                                                &lock_timeout_seconds, error))
+          return FALSE;
+
+        self->lock_timeout_seconds = g_ascii_strtoull (lock_timeout_seconds, NULL, 10);
+      }
+  }
+
   { g_autofree char *compression_level_str = NULL;
 
     /* gzip defaults to 6 */
@@ -3388,6 +3941,115 @@ ostree_repo_delete_object (OstreeRepo           *self,
   return TRUE;
 }
 
+static gboolean
+fsck_metadata_object (OstreeRepo           *self,
+                      OstreeObjectType      objtype,
+                      const char           *sha256,
+                      GCancellable         *cancellable,
+                      GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck ", sha256, ".", ostree_object_type_to_string (objtype));
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GVariant) metadata = NULL;
+  if (!load_metadata_internal (self, objtype, sha256, TRUE,
+                               &metadata, NULL, NULL, NULL,
+                               cancellable, error))
+    return FALSE;
+
+  g_auto(OtChecksum) hasher = { 0, };
+  ot_checksum_init (&hasher);
+  ot_checksum_update (&hasher, g_variant_get_data (metadata), g_variant_get_size (metadata));
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ot_checksum_get_hexdigest (&hasher, actual_checksum, sizeof (actual_checksum));
+  if (!_ostree_compare_object_checksum (objtype, sha256, actual_checksum, error))
+    return FALSE;
+
+  switch (objtype)
+    {
+    case OSTREE_OBJECT_TYPE_COMMIT:
+      if (!ostree_validate_structureof_commit (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_TREE:
+      if (!ostree_validate_structureof_dirtree (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_DIR_META:
+      if (!ostree_validate_structureof_dirmeta (metadata, error))
+        return FALSE;
+      break;
+    case OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT:
+    case OSTREE_OBJECT_TYPE_COMMIT_META:
+      /* TODO */
+      break;
+    case OSTREE_OBJECT_TYPE_FILE:
+      g_assert_not_reached ();
+      break;
+    }
+
+  return TRUE;
+}
+
+static gboolean
+fsck_content_object (OstreeRepo           *self,
+                     const char           *sha256,
+                     GCancellable         *cancellable,
+                     GError              **error)
+{
+  const char *errmsg = glnx_strjoina ("fsck content object ", sha256);
+  GLNX_AUTO_PREFIX_ERROR (errmsg, error);
+  g_autoptr(GInputStream) input = NULL;
+  g_autoptr(GFileInfo) file_info = NULL;
+  g_autoptr(GVariant) xattrs = NULL;
+
+  if (!ostree_repo_load_file (self, sha256, &input, &file_info, &xattrs,
+                              cancellable, error))
+    return FALSE;
+
+  /* TODO more consistency checks here */
+  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+  if (!ostree_validate_structureof_file_mode (mode, error))
+    return FALSE;
+
+  g_autofree guchar *computed_csum = NULL;
+  if (!ostree_checksum_file_from_input (file_info, xattrs, input,
+                                        OSTREE_OBJECT_TYPE_FILE, &computed_csum,
+                                        cancellable, error))
+    return FALSE;
+
+  char actual_checksum[OSTREE_SHA256_STRING_LEN+1];
+  ostree_checksum_inplace_from_bytes (computed_csum, actual_checksum);
+  return _ostree_compare_object_checksum (OSTREE_OBJECT_TYPE_FILE, sha256, actual_checksum, error);
+}
+
+/**
+ * ostree_repo_fsck_object:
+ * @self: Repo
+ * @objtype: Object type
+ * @sha256: Checksum
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Verify consistency of the object; this performs checks only relevant to the
+ * immediate object itself, such as checksumming. This API call will not itself
+ * traverse metadata objects for example.
+ *
+ * Since: 2017.15
+ */
+gboolean
+ostree_repo_fsck_object (OstreeRepo           *self,
+                         OstreeObjectType      objtype,
+                         const char           *sha256,
+                         GCancellable         *cancellable,
+                         GError              **error)
+{
+  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+    return fsck_metadata_object (self, objtype, sha256, cancellable, error);
+  else
+    return fsck_content_object (self, sha256, cancellable, error);
+}
+
 /**
  * ostree_repo_import_object_from:
  * @self: Destination repo

src/libostree/ostree-repo.h

@@ -107,6 +107,48 @@ OstreeRepo *  ostree_repo_create_at (int             dfd,
 
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
 
+/**
+ * OstreeRepoLockType:
+ * @OSTREE_REPO_LOCK_SHARED: A shared lock
+ * @OSTREE_REPO_LOCK_EXCLUSIVE: An exclusive lock
+ *
+ * The type of repository lock to acquire.
+ *
+ * Since: 2017.14
+ */
+typedef enum {
+  OSTREE_REPO_LOCK_SHARED,
+  OSTREE_REPO_LOCK_EXCLUSIVE
+} OstreeRepoLockType;
+
+_OSTREE_PUBLIC
+gboolean      ostree_repo_lock_push (OstreeRepo          *self,
+                                     OstreeRepoLockType   lock_type,
+                                     GCancellable        *cancellable,
+                                     GError             **error);
+_OSTREE_PUBLIC
+gboolean      ostree_repo_lock_pop (OstreeRepo    *self,
+                                    GCancellable  *cancellable,
+                                    GError       **error);
+
+/**
+ * OstreeRepoAutoLock: (skip)
+ *
+ * This is simply an alias to #OstreeRepo used for automatic lock cleanup.
+ * See ostree_repo_auto_lock_push() for its intended usage.
+ *
+ * Since: 2017.14
+ */
+typedef OstreeRepo OstreeRepoAutoLock;
+
+_OSTREE_PUBLIC
+OstreeRepoAutoLock * ostree_repo_auto_lock_push (OstreeRepo          *self,
+                                                 OstreeRepoLockType   lock_type,
+                                                 GCancellable        *cancellable,
+                                                 GError             **error);
+_OSTREE_PUBLIC
+void          ostree_repo_auto_lock_cleanup (OstreeRepoAutoLock *lock);
+
 _OSTREE_PUBLIC
 const gchar * ostree_repo_get_collection_id (OstreeRepo   *self);
 _OSTREE_PUBLIC
@@ -320,6 +362,12 @@ gboolean      ostree_repo_abort_transaction (OstreeRepo     *self,
                                              GCancellable   *cancellable,
                                              GError        **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_mark_commit_partial (OstreeRepo     *self,
+                                               const char     *checksum,
+                                               gboolean        is_partial,
+                                               GError        **error);
+
 _OSTREE_PUBLIC
 void          ostree_repo_transaction_set_refspec (OstreeRepo *self,
                                                    const char *refspec,
@@ -612,6 +660,13 @@ gboolean      ostree_repo_delete_object (OstreeRepo           *self,
                                          GCancellable         *cancellable,
                                          GError              **error);
 
+_OSTREE_PUBLIC
+gboolean      ostree_repo_fsck_object (OstreeRepo           *self,
+                                       OstreeObjectType      objtype,
+                                       const char           *sha256,
+                                       GCancellable         *cancellable,
+                                       GError              **error);
+
 /** 
  * OstreeRepoCommitFilterResult:
  * @OSTREE_REPO_COMMIT_FILTER_ALLOW: Do commit this object

src/libostree/ostree-version.h

@@ -43,7 +43,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_RELEASE_VERSION (14)
+#define OSTREE_RELEASE_VERSION (15)
 
 /**
  * OSTREE_VERSION
@@ -52,7 +52,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION (2017.14)
+#define OSTREE_VERSION (2017.15)
 
 /**
  * OSTREE_VERSION_S:
@@ -62,7 +62,7 @@
  *
  * Since: 2017.4
  */
-#define OSTREE_VERSION_S "2017.14"
+#define OSTREE_VERSION_S "2017.15"
 
 #define OSTREE_ENCODE_VERSION(year,release) \
         ((year) << 16 | (release))

src/libotutil/ot-checksum-instream.c

@@ -65,6 +65,16 @@ ot_checksum_instream_init (OtChecksumInstream *self)
 OtChecksumInstream *
 ot_checksum_instream_new (GInputStream    *base,
                           GChecksumType    checksum_type)
+{
+  return ot_checksum_instream_new_with_start (base, checksum_type, NULL, 0);
+}
+
+/* Initialize a checksum stream with starting state from data */
+OtChecksumInstream *
+ot_checksum_instream_new_with_start (GInputStream   *base,
+                                     GChecksumType   checksum_type,
+                                     const guint8   *buf,
+                                     size_t          len)
 {
   OtChecksumInstream *stream;
 
@@ -77,6 +87,8 @@ ot_checksum_instream_new (GInputStream    *base,
   /* For now */
   g_assert (checksum_type == G_CHECKSUM_SHA256);
   ot_checksum_init (&stream->priv->checksum);
+  if (buf)
+    ot_checksum_update (&stream->priv->checksum, buf, len);
 
   return (OtChecksumInstream*) (stream);
 }

src/libotutil/ot-checksum-instream.h

@@ -51,6 +51,8 @@ struct _OtChecksumInstreamClass
 GType          ot_checksum_instream_get_type     (void) G_GNUC_CONST;
 
 OtChecksumInstream * ot_checksum_instream_new          (GInputStream   *stream, GChecksumType   checksum);
+OtChecksumInstream * ot_checksum_instream_new_with_start (GInputStream   *stream, GChecksumType   checksum,
+                                                          const guint8 *buf, size_t len);
 
 char * ot_checksum_instream_get_string (OtChecksumInstream *stream);
 

src/ostree/ot-builtin-commit.c

@@ -37,11 +37,13 @@ static char *opt_body_file;
 static gboolean opt_editor;
 static char *opt_parent;
 static gboolean opt_orphan;
+static gboolean opt_no_bindings;
 static char **opt_bind_refs;
 static char *opt_branch;
 static char *opt_statoverride_file;
 static char *opt_skiplist_file;
 static char **opt_metadata_strings;
+static char **opt_metadata_variants;
 static char **opt_detached_metadata_strings;
 static gboolean opt_link_checkout_speedup;
 static gboolean opt_skip_if_unchanged;
@@ -89,9 +91,11 @@ static GOptionEntry options[] = {
   { "editor", 'e', 0, G_OPTION_ARG_NONE, &opt_editor, "Use an editor to write the commit message", NULL },
   { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" },
   { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL },
+  { "no-bindings", 0, 0, G_OPTION_ARG_NONE, &opt_no_bindings, "Do not write any ref bindings", NULL },
   { "bind-ref", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_bind_refs, "Add a ref to ref binding commit metadata", "BRANCH" },
   { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "dir=PATH or tar=TARFILE or ref=COMMIT" },
   { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Add a key/value pair to metadata", "KEY=VALUE" },
+  { "add-metadata", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_variants, "Add a key/value pair to metadata, where the KEY is a string, an VALUE is g_variant_parse() formatted", "KEY=VALUE" },
   { "add-detached-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_detached_metadata_strings, "Add a key/value pair to detached metadata", "KEY=VALUE" },
   { "owner-uid", 0, 0, G_OPTION_ARG_INT, &opt_owner_uid, "Set file ownership user id", "UID" },
   { "owner-gid", 0, 0, G_OPTION_ARG_INT, &opt_owner_gid, "Set file ownership group id", "GID" },
@@ -321,13 +325,11 @@ commit_editor (OstreeRepo     *repo,
 }
 
 static gboolean
-parse_keyvalue_strings (char             **strings,
+parse_keyvalue_strings (GVariantBuilder   *builder,
-                        GVariant         **out_metadata,
+                        char             **strings,
+                        gboolean           is_gvariant_print,
                         GError           **error)
 {
-  g_autoptr(GVariantBuilder) builder =
-    g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
-
   for (char ** iter = strings; *iter; iter++)
     {
       const char *s = *iter;
@@ -335,11 +337,19 @@ parse_keyvalue_strings (char             **strings,
       if (!eq)
         return glnx_throw (error, "Missing '=' in KEY=VALUE metadata '%s'", s);
       g_autofree char *key = g_strndup (s, eq - s);
+      if (is_gvariant_print)
+        {
+          g_autoptr(GVariant) value = g_variant_parse (NULL, eq + 1, NULL, NULL, error);
+          if (!value)
+            return glnx_prefix_error (error, "Parsing %s", s);
+
+          g_variant_builder_add (builder, "{sv}", key, value);
+        }
+      else
         g_variant_builder_add (builder, "{sv}", key,
                                g_variant_new_string (eq + 1));
     }
 
-  *out_metadata = g_variant_ref_sink (g_variant_builder_end (builder));
   return TRUE;
 }
 
@@ -370,12 +380,10 @@ compare_strings (gconstpointer a, gconstpointer b)
 static void
 add_ref_binding (GVariantBuilder *metadata_builder)
 {
-  if (opt_orphan)
+  g_assert (opt_branch != NULL || opt_orphan);
-    return;
-
-  g_assert_nonnull (opt_branch);
 
   g_autoptr(GPtrArray) refs = g_ptr_array_new ();
+  if (opt_branch != NULL)
     g_ptr_array_add (refs, opt_branch);
   for (char **iter = opt_bind_refs; iter != NULL && *iter != NULL; ++iter)
     g_ptr_array_add (refs, *iter);
@@ -458,17 +466,31 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio
         goto out;
     }
 
-  if (opt_metadata_strings)
+  if (opt_metadata_strings || opt_metadata_variants)
     {
-      if (!parse_keyvalue_strings (opt_metadata_strings,
+      g_autoptr(GVariantBuilder) builder =
-                                   &metadata, error))
+        g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+
+      if (opt_metadata_strings &&
+          !parse_keyvalue_strings (builder, opt_metadata_strings, FALSE, error))
           goto out;
+
+      if (opt_metadata_variants &&
+          !parse_keyvalue_strings (builder, opt_metadata_variants, TRUE, error))
+        goto out;
+
+      metadata = g_variant_ref_sink (g_variant_builder_end (builder));
     }
+
   if (opt_detached_metadata_strings)
     {
-      if (!parse_keyvalue_strings (opt_detached_metadata_strings,
+      g_autoptr(GVariantBuilder) builder =
-                                   &detached_metadata, error))
+        g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+
+      if (!parse_keyvalue_strings (builder, opt_detached_metadata_strings, FALSE, error))
         goto out;
+
+      detached_metadata = g_variant_ref_sink (g_variant_builder_end (builder));
     }
 
   if (!(opt_branch || opt_orphan))
@@ -725,9 +747,12 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio
     {
       gboolean update_summary;
       guint64 timestamp;
-      g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
 
+      if (!opt_no_bindings)
+        {
+          g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata);
           fill_bindings (repo, old_metadata, &metadata);
+        }
 
       if (!opt_timestamp)
         {

src/ostree/ot-builtin-fsck.c

@@ -30,6 +30,8 @@
 static gboolean opt_quiet;
 static gboolean opt_delete;
 static gboolean opt_add_tombstones;
+static gboolean opt_verify_bindings;
+static gboolean opt_verify_back_refs;
 
 /* ATTENTION:
  * Please remember to update the bash-completion script (bash/ostree) and
@@ -40,122 +42,42 @@ static GOptionEntry options[] = {
   { "add-tombstones", 0, 0, G_OPTION_ARG_NONE, &opt_add_tombstones, "Add tombstones for missing commits", NULL },
   { "quiet", 'q', 0, G_OPTION_ARG_NONE, &opt_quiet, "Only print error messages", NULL },
   { "delete", 0, 0, G_OPTION_ARG_NONE, &opt_delete, "Remove corrupted objects", NULL },
+  { "verify-bindings", 0, 0, G_OPTION_ARG_NONE, &opt_verify_bindings, "Verify ref bindings", NULL },
+  { "verify-back-refs", 0, 0, G_OPTION_ARG_NONE, &opt_verify_back_refs, "Verify back-references (implies --verify-bindings)", NULL },
   { NULL }
 };
 
 static gboolean
-load_and_fsck_one_object (OstreeRepo            *repo,
+fsck_one_object (OstreeRepo            *repo,
                  const char            *checksum,
                  OstreeObjectType       objtype,
                  gboolean              *out_found_corruption,
                  GCancellable          *cancellable,
                  GError               **error)
 {
-  gboolean missing = FALSE;
-  g_autoptr(GVariant) metadata = NULL;
-  g_autoptr(GInputStream) input = NULL;
-  g_autoptr(GFileInfo) file_info = NULL;
-  g_autoptr(GVariant) xattrs = NULL;
   g_autoptr(GError) temp_error = NULL;
-
+  if (!ostree_repo_fsck_object (repo, objtype, checksum, cancellable, &temp_error))
-  if (OSTREE_OBJECT_TYPE_IS_META (objtype))
-    {
-      if (!ostree_repo_load_variant (repo, objtype,
-                                     checksum, &metadata, &temp_error))
     {
       if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
           g_clear_error (&temp_error);
           g_printerr ("Object missing: %s.%s\n", checksum,
                       ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading metadata object %s", checksum);
-            }
-        }
-      else
-        {
-          if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
-            {
-              if (!ostree_validate_structureof_commit (metadata, error))
-                return glnx_prefix_error (error, "While validating commit metadata '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE)
-            {
-              if (!ostree_validate_structureof_dirtree (metadata, error))
-                return glnx_prefix_error (error, "While validating directory tree '%s'", checksum);
-            }
-          else if (objtype == OSTREE_OBJECT_TYPE_DIR_META)
-            {
-              if (!ostree_validate_structureof_dirmeta (metadata, error))
-                return glnx_prefix_error (error, "While validating directory metadata '%s'", checksum);
-            }
-
-          input = g_memory_input_stream_new_from_data (g_variant_get_data (metadata),
-                                                       g_variant_get_size (metadata),
-                                                       NULL);
-
-        }
-    }
-  else
-    {
-      guint32 mode;
-      g_assert (objtype == OSTREE_OBJECT_TYPE_FILE);
-      if (!ostree_repo_load_file (repo, checksum, &input, &file_info,
-                                  &xattrs, cancellable, &temp_error))
-        {
-          if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (&temp_error);
-              g_printerr ("Object missing: %s.%s\n", checksum,
-                          ostree_object_type_to_string (objtype));
-              missing = TRUE;
-            }
-          else
-            {
-              g_propagate_error (error, g_steal_pointer (&temp_error));
-              return glnx_prefix_error (error, "Loading file object %s", checksum);
-            }
-        }
-      else
-        {
-          mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
-          if (!ostree_validate_structureof_file_mode (mode, error))
-            return glnx_prefix_error (error, "While validating file '%s'", checksum);
-        }
-    }
-
-  if (missing)
-    {
           *out_found_corruption = TRUE;
         }
       else
         {
-      g_autofree guchar *computed_csum = NULL;
-      g_autofree char *tmp_checksum = NULL;
-
-      if (!ostree_checksum_file_from_input (file_info, xattrs, input,
-                                            objtype, &computed_csum,
-                                            cancellable, error))
-        return FALSE;
-
-      tmp_checksum = ostree_checksum_from_bytes (computed_csum);
-      if (strcmp (checksum, tmp_checksum) != 0)
-        {
-          g_autofree char *msg = g_strdup_printf ("corrupted object %s.%s; actual checksum: %s",
-                                               checksum, ostree_object_type_to_string (objtype),
-                                               tmp_checksum);
           if (opt_delete)
             {
-              g_printerr ("%s\n", msg);
+              g_printerr ("%s\n", temp_error->message);
               (void) ostree_repo_delete_object (repo, objtype, checksum, cancellable, NULL);
               *out_found_corruption = TRUE;
             }
           else
-            return glnx_throw (error, "%s", msg);
+            {
+              g_propagate_error (error, g_steal_pointer (&temp_error));
+              return FALSE;
+            }
         }
     }
 
@@ -189,8 +111,10 @@ fsck_reachable_objects_from_commits (OstreeRepo            *repo,
         return FALSE;
     }
 
+  g_auto(GLnxConsoleRef) console = { 0, };
+  glnx_console_lock (&console);
+
   const guint count = g_hash_table_size (reachable_objects);
-  const guint mod = count / 10;
   guint i = 0;
   g_hash_table_iter_init (&hash_iter, reachable_objects);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
@@ -201,13 +125,50 @@ fsck_reachable_objects_from_commits (OstreeRepo            *repo,
 
       ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
 
-      if (!load_and_fsck_one_object (repo, checksum, objtype, out_found_corruption,
+      if (!fsck_one_object (repo, checksum, objtype, out_found_corruption,
                             cancellable, error))
         return FALSE;
 
-      if (mod == 0 || (i % mod == 0))
-        g_print ("%u/%u objects\n", i + 1, count);
       i++;
+      glnx_console_progress_n_items ("fsck objects", i, count);
+    }
+
+  return TRUE;
+}
+
+/* Check that a given commit object is valid for the ref it was looked up via.
+ * @collection_id will be %NULL for normal refs, and non-%NULL for collection–refs. */
+static gboolean
+fsck_commit_for_ref (OstreeRepo    *repo,
+                     const char    *checksum,
+                     const char    *collection_id,
+                     const char    *ref_name,
+                     gboolean      *found_corruption,
+                     GCancellable  *cancellable,
+                     GError       **error)
+{
+  if (!fsck_one_object (repo, checksum, OSTREE_OBJECT_TYPE_COMMIT,
+                        found_corruption,
+                        cancellable, error))
+    return FALSE;
+
+  /* Check the commit exists. */
+  g_autoptr(GVariant) commit = NULL;
+  if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
+                                 checksum, &commit, error))
+    {
+      if (collection_id != NULL)
+        return glnx_prefix_error (error, "Loading commit for ref (%s, %s)",
+                                  collection_id, ref_name);
+      else
+        return glnx_prefix_error (error, "Loading commit for ref %s", ref_name);
+    }
+
+  /* Check its bindings. */
+  if (opt_verify_bindings)
+    {
+      if (!ostree_cmd__private__ ()->ostree_repo_verify_bindings (collection_id, ref_name, commit, error))
+        return glnx_prefix_error (error, "Commit %s", checksum);
     }
 
   return TRUE;
@@ -237,12 +198,14 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
   g_hash_table_iter_init (&hash_iter, all_refs);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
     {
-      const char *refname = key;
+      const char *refspec = key;
       const char *checksum = value;
-      g_autoptr(GVariant) commit = NULL;
+      g_autofree char *ref_name = NULL;
-      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
+      if (!ostree_parse_refspec (refspec, NULL, &ref_name, error))
-                                     checksum, &commit, error))
+        return FALSE;
-        return glnx_prefix_error (error, "Loading commit for ref %s", refname);
+      if (!fsck_commit_for_ref (repo, checksum, NULL, ref_name,
+                                &found_corruption, cancellable, error))
+        return FALSE;
     }
 
 #ifdef OSTREE_ENABLE_EXPERIMENTAL_API
@@ -259,12 +222,9 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
     {
       const OstreeCollectionRef *ref = key;
-      const char *checksum = value;
+      if (!fsck_commit_for_ref (repo, value, ref->collection_id, ref->ref_name,
-      g_autoptr(GVariant) commit = NULL;
+                                &found_corruption, cancellable, error))
-      if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT,
+        return FALSE;
-                                     checksum, &commit, error))
-        return glnx_prefix_error (error, "Loading commit for ref (%s, %s)",
-                                  ref->collection_id, ref->ref_name);
     }
 #endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
 
@@ -284,6 +244,9 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
   if (opt_add_tombstones)
     tombstones = g_ptr_array_new_with_free_func (g_free);
 
+  if (opt_verify_back_refs)
+    opt_verify_bindings = TRUE;
+
   guint n_partial = 0;
   g_hash_table_iter_init (&hash_iter, objects);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
@@ -301,6 +264,77 @@ ostree_builtin_fsck (int argc, char **argv, OstreeCommandInvocation *invocation,
           if (!ostree_repo_load_commit (repo, checksum, &commit, &commitstate, error))
             return FALSE;
 
+          /* If requested, check that all the refs listed in the ref-bindings
+           * for this commit resolve back to this commit. */
+          if (opt_verify_back_refs)
+            {
+              g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0);
+
+              const char *collection_id = NULL;
+#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+              if (!g_variant_lookup (metadata,
+                                     OSTREE_COMMIT_META_KEY_COLLECTION_BINDING,
+                                     "&s",
+                                     &collection_id))
+                collection_id = NULL;
+#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
+
+              g_autofree const char **refs = NULL;
+              if (g_variant_lookup (metadata,
+                                    OSTREE_COMMIT_META_KEY_REF_BINDING,
+                                    "^a&s",
+                                    &refs))
+                {
+                  for (const char **iter = refs; *iter != NULL; ++iter)
+                    {
+                      g_autofree char *checksum_for_ref = NULL;
+
+#ifdef OSTREE_ENABLE_EXPERIMENTAL_API
+                      if (collection_id != NULL)
+                        {
+                          const OstreeCollectionRef collection_ref = { (char *) collection_id, (char *) *iter };
+                          if (!ostree_repo_resolve_collection_ref (repo, &collection_ref,
+                                                                   TRUE,
+                                                                   OSTREE_REPO_RESOLVE_REV_EXT_NONE,
+                                                                   &checksum_for_ref,
+                                                                   cancellable,
+                                                                   error))
+                            return FALSE;
+                        }
+                      else
+#endif  /* OSTREE_ENABLE_EXPERIMENTAL_API */
+                        {
+                          if (!ostree_repo_resolve_rev (repo, *iter, TRUE,
+                                                        &checksum_for_ref, error))
+                            return FALSE;
+                        }
+
+                      if (checksum_for_ref == NULL)
+                        {
+                          if (collection_id != NULL)
+                            return glnx_throw (error,
+                                               "Collection–ref (%s, %s) in bindings for commit %s does not exist",
+                                               collection_id, *iter, checksum);
+                          else
+                            return glnx_throw (error,
+                                               "Ref ‘%s’ in bindings for commit %s does not exist",
+                                               *iter, checksum);
+                        }
+                      else if (g_strcmp0 (checksum_for_ref, checksum) != 0)
+                        {
+                          if (collection_id != NULL)
+                            return glnx_throw (error,
+                                               "Collection–ref (%s, %s) in bindings for commit %s does not resolve to that commit",
+                                               collection_id, *iter, checksum);
+                          else
+                            return glnx_throw (error,
+                                               "Ref ‘%s’ in bindings for commit %s does not resolve to that commit",
+                                               *iter, checksum);
+                        }
+                    }
+                }
+            }
+
           if (opt_add_tombstones)
             {
               GError *local_error = NULL;

src/ostree/ot-builtin-refs.c

@@ -223,6 +223,8 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab
 
       if (opt_alias)
         {
+          if (remote)
+            return glnx_throw (error, "Cannot create alias to remote ref: %s", remote);
           if (!ostree_repo_set_alias_ref_immediate (repo, remote, ref, refspec_prefix,
                                                     cancellable, error))
             goto out;

src/ostree/ot-builtin-show.c

@@ -32,6 +32,7 @@ static char* opt_print_variant_type;
 static char* opt_print_metadata_key;
 static char* opt_print_detached_metadata_key;
 static gboolean opt_raw;
+static gboolean opt_no_byteswap;
 static char *opt_gpg_homedir;
 static char *opt_gpg_verify_remote;
 
@@ -46,6 +47,7 @@ static GOptionEntry options[] = {
   { "print-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_metadata_key, "Print string value of metadata key", "KEY" },
   { "print-detached-metadata-key", 0, 0, G_OPTION_ARG_STRING, &opt_print_detached_metadata_key, "Print string value of detached metadata key", "KEY" },
   { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data" },
+  { "no-byteswap", 'B', 0, G_OPTION_ARG_NONE, &opt_no_byteswap, "Do not automatically convert variant data from big endian" },
   { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"},
   { "gpg-verify-remote", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_verify_remote, "Use REMOTE name for GPG configuration", "REMOTE"},
   { NULL }
@@ -132,6 +134,12 @@ do_print_metadata_key (OstreeRepo     *repo,
       return FALSE;
     }
 
+  if (opt_no_byteswap)
+    {
+      g_autofree char *formatted = g_variant_print (value, TRUE);
+      g_print ("%s\n", formatted);
+    }
+  else
     ot_dump_variant (value);
   return TRUE;
 }
@@ -150,6 +158,8 @@ print_object (OstreeRepo          *repo,
     return FALSE;
   if (opt_raw)
     flags |= OSTREE_DUMP_RAW;
+  if (opt_no_byteswap)
+    flags |= OSTREE_DUMP_UNSWAPPED;
   ot_dump_object (objtype, checksum, variant, flags);
 
   if (objtype == OSTREE_OBJECT_TYPE_COMMIT)

src/ostree/ot-dump.c

@@ -157,7 +157,12 @@ ot_dump_object (OstreeObjectType   objtype,
 {
   g_print ("%s %s\n", ostree_object_type_to_string (objtype), checksum);
 
-  if (flags & OSTREE_DUMP_RAW)
+  if (flags & OSTREE_DUMP_UNSWAPPED)
+    {
+      g_autofree char *formatted = g_variant_print (variant, TRUE);
+      g_print ("%s\n", formatted);
+    }
+  else if (flags & OSTREE_DUMP_RAW)
     {
       ot_dump_variant (variant);
       return;

src/ostree/ot-dump.h

@@ -26,8 +26,9 @@
 #include "ostree-core.h"
 
 typedef enum {
-  OSTREE_DUMP_NONE = 0,
+  OSTREE_DUMP_NONE = (1 << 0),
-  OSTREE_DUMP_RAW = 1,
+  OSTREE_DUMP_RAW = (1 << 1),
+  OSTREE_DUMP_UNSWAPPED = (1 << 2),
 } OstreeDumpFlags;
 
 void   ot_dump_variant    (GVariant *variant);

tests/basic-test.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..$((77 + ${extra_basic_tests:-0}))"
+echo "1..$((78 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -751,16 +751,32 @@ $OSTREE commit ${COMMIT_ARGS} -s sometest -b test2 checkout-test2
 echo "ok commit with directory filename"
 
 cd $test_tmpdir/checkout-test2
-$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Metadata string" --add-metadata-string=FOO=BAR --add-metadata-string=KITTENS=CUTE --add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=test2
+$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Metadata string" --add-metadata-string=FOO=BAR \
+        --add-metadata-string=KITTENS=CUTE --add-detached-metadata-string=SIGNATURE=HANCOCK \
+        --add-metadata=SOMENUM='uint64 42' --tree=ref=test2
 cd ${test_tmpdir}
 $OSTREE show --print-metadata-key=FOO test2 > test2-meta
 assert_file_has_content test2-meta "BAR"
 $OSTREE show --print-metadata-key=KITTENS test2 > test2-meta
 assert_file_has_content test2-meta "CUTE"
+$OSTREE show --print-metadata-key=SOMENUM test2 > test2-meta
+assert_file_has_content test2-meta "uint64 3026418949592973312"
+$OSTREE show -B --print-metadata-key=SOMENUM test2 > test2-meta
+assert_file_has_content test2-meta "uint64 42"
 $OSTREE show --print-detached-metadata-key=SIGNATURE test2 > test2-meta
 assert_file_has_content test2-meta "HANCOCK"
 echo "ok metadata commit with strings"
 
+cd ${test_tmpdir}
+$OSTREE show --print-metadata-key=ostree.ref-binding test2 > test2-ref-binding
+assert_file_has_content test2-ref-binding 'test2'
+
+$OSTREE commit ${COMMIT_ARGS} -b test2-unbound --no-bindings --tree=dir=${test_tmpdir}/checkout-test2
+if $OSTREE show --print-metadata-key=ostree.ref-binding; then
+    fatal "ref bindings found with --no-bindings?"
+fi
+echo "ok refbinding"
+
 if ! skip_one_without_user_xattrs; then
     cd ${test_tmpdir}
     rm repo2 -rf

tests/libtest-core.sh

@@ -37,6 +37,8 @@ assert_not_reached () {
 # (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8)
 if locale -a | grep C.UTF-8 >/dev/null; then
     export LC_ALL=C.UTF-8
+elif locale -a | grep C.utf8 >/dev/null; then
+    export LC_ALL=C.utf8
 else
     export LC_ALL=C
 fi

tests/pull-test.sh

@@ -183,7 +183,7 @@ if ! skip_one_without_user_xattrs; then
         if ${CMD_PREFIX} ostree --repo=cacherepo fsck 2>err.txt; then
             fatal "corrupt repo fsck?"
         fi
-        assert_file_has_content err.txt "corrupted.*${checksum}"
+        assert_file_has_content err.txt "Corrupted.*${checksum}"
         rm ostree-srv/corruptrepo -rf
         ostree_repo_init ostree-srv/corruptrepo --mode=archive
         ${CMD_PREFIX} ostree --repo=ostree-srv/corruptrepo pull-local cacherepo main

tests/test-basic-c.c

@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <gio/gio.h>
 #include <string.h>
+#include <err.h>
 
 #include "libglnx.h"
 #include "libostreetest.h"
@@ -236,6 +237,72 @@ test_object_writes (gconstpointer data)
   }
 }
 
+static gboolean
+impl_test_break_hardlink (int tmp_dfd,
+                          const char *path,
+                          GError **error)
+{
+  const char *linkedpath = glnx_strjoina (path, ".link");
+  struct stat orig_stbuf;
+  if (!glnx_fstatat (tmp_dfd, path, &orig_stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+
+  /* Calling ostree_break_hardlink() should be a noop */
+  struct stat stbuf;
+  if (!ostree_break_hardlink (tmp_dfd, path, TRUE, NULL, error))
+    return FALSE;
+  if (!glnx_fstatat (tmp_dfd, path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+
+  g_assert_cmpint (orig_stbuf.st_dev, ==, stbuf.st_dev);
+  g_assert_cmpint (orig_stbuf.st_ino, ==, stbuf.st_ino);
+
+  if (linkat (tmp_dfd, path, tmp_dfd, linkedpath, 0) < 0)
+    return glnx_throw_errno_prefix (error, "linkat");
+
+  if (!ostree_break_hardlink (tmp_dfd, path, TRUE, NULL, error))
+    return FALSE;
+  if (!glnx_fstatat (tmp_dfd, path, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  /* This file should be different */
+  g_assert_cmpint (orig_stbuf.st_dev, ==, stbuf.st_dev);
+  g_assert_cmpint (orig_stbuf.st_ino, !=, stbuf.st_ino);
+  /* But this one is still the same */
+  if (!glnx_fstatat (tmp_dfd, linkedpath, &stbuf, AT_SYMLINK_NOFOLLOW, error))
+    return FALSE;
+  g_assert_cmpint (orig_stbuf.st_dev, ==, stbuf.st_dev);
+  g_assert_cmpint (orig_stbuf.st_ino, ==, stbuf.st_ino);
+
+  (void) unlinkat (tmp_dfd, path, 0);
+  (void) unlinkat (tmp_dfd, linkedpath, 0);
+
+  return TRUE;
+}
+
+static void
+test_break_hardlink (void)
+{
+  int tmp_dfd = AT_FDCWD;
+  g_autoptr(GError) error = NULL;
+
+  /* Regular file */
+  const char hello_hardlinked_content[] = "hello hardlinked content";
+  glnx_file_replace_contents_at (tmp_dfd, "test-hardlink",
+                                 (guint8*)hello_hardlinked_content,
+                                 strlen (hello_hardlinked_content),
+                                 GLNX_FILE_REPLACE_NODATASYNC,
+                                 NULL, &error);
+  g_assert_no_error (error);
+  (void)impl_test_break_hardlink (tmp_dfd, "test-hardlink", &error);
+  g_assert_no_error (error);
+
+  /* Symlink */
+  if (symlinkat ("some-path", tmp_dfd, "test-symhardlink") < 0)
+    err (1, "symlinkat");
+  (void)impl_test_break_hardlink (tmp_dfd, "test-symhardlink", &error);
+  g_assert_no_error (error);
+}
+
 static GVariant*
 xattr_cb (OstreeRepo  *repo,
           const char  *path,
@@ -376,6 +443,7 @@ int main (int argc, char **argv)
   g_test_add_data_func ("/raw-file-to-archive-stream", repo, test_raw_file_to_archive_stream);
   g_test_add_data_func ("/objectwrites", repo, test_object_writes);
   g_test_add_func ("/xattrs-devino-cache", test_devino_cache_xattrs);
+  g_test_add_func ("/break-hardlink", test_break_hardlink);
   g_test_add_func ("/remotename", test_validate_remotename);
 
   return g_test_run();

tests/test-concurrency.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2017 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+from __future__ import print_function
+import os
+import sys
+import shutil
+import subprocess
+from multiprocessing import cpu_count
+
+def fatal(msg):
+    sys.stderr.write(msg)
+    sys.stderr.write('\n')
+    sys.exit(1)
+
+# Create 20 files with content based on @dname + a serial, basically to have
+# different files with different checksums.
+def mktree(dname, serial=0):
+    print('Creating tree', dname, file=sys.stderr)
+    os.mkdir(dname, 0755)
+    for v in xrange(20):
+        with open('{}/{}'.format(dname, v), 'w') as f:
+            f.write('{} {} {}\n'.format(dname, serial, v))
+
+subprocess.check_call(['ostree', '--repo=repo', 'init', '--mode=bare'])
+# like the bit in libtest, but let's do it unconditionally since it's simpler,
+# and we don't need xattr coverage for this
+with open('repo/config', 'a') as f:
+    f.write('disable-xattrs=true\n')
+    f.write('locking=true\n')
+
+def commit(v):
+    tdir='tree{}'.format(v)
+    cmd = ['ostree', '--repo=repo', 'commit', '--fsync=0', '-b', tdir, '--tree=dir='+tdir]
+    proc = subprocess.Popen(cmd)
+    print('PID {}'.format(proc.pid), *cmd, file=sys.stderr)
+    return proc
+def prune():
+    cmd = ['ostree', '--repo=repo', 'prune', '--refs-only']
+    proc = subprocess.Popen(cmd)
+    print('PID {}:'.format(proc.pid), *cmd, file=sys.stderr)
+    return proc
+
+def wait_check(proc):
+    pid = proc.pid
+    proc.wait()
+    if proc.returncode != 0:
+        sys.stderr.write("process {} exited with code {}\n".format(proc.pid, proc.returncode))
+        return False
+    else:
+        sys.stderr.write('PID {} exited OK\n'.format(pid))
+        return True
+
+print("1..2")
+
+def run(n_committers, n_pruners):
+    # The number of committers needs to be even since we only create half as
+    # many trees
+    n_committers += n_committers % 2
+
+    committers = set()
+    pruners = set()
+
+    print('n_committers', n_committers, 'n_pruners', n_pruners, file=sys.stderr)
+    n_trees = n_committers / 2
+    for v in xrange(n_trees):
+        mktree('tree{}'.format(v))
+
+    for v in xrange(n_committers):
+        committers.add(commit(v / 2))
+    for v in xrange(n_pruners):
+        pruners.add(prune())
+
+    failed = False
+    for committer in committers:
+        if not wait_check(committer):
+            failed = True
+    for pruner in pruners:
+        if not wait_check(pruner):
+            failed = True
+    if failed:
+        fatal('A child process exited abnormally')
+
+    for v in xrange(n_trees):
+        shutil.rmtree('tree{}'.format(v))
+
+# No concurrent pruning
+run(cpu_count()/2 + 2, 0)
+print("ok no concurrent prunes")
+
+run(cpu_count()/2 + 4, 3)
+print("ok concurrent prunes")

tests/test-corruption.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+# Copyright (C) 2011,2017 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..3"
+echo "1..4"
 
 . $(dirname $0)/libtest.sh
 
@@ -29,12 +29,26 @@ setup_test_repository "bare"
 $OSTREE checkout test2 checkout-test2
 cd checkout-test2
 chmod o+x firstfile
-$OSTREE fsck -q && (echo 1>&2 "fsck unexpectedly succeeded"; exit 1)
+if $OSTREE fsck -q; then
+    fatal "fsck unexpectedly succeeded"
+fi
 chmod o-x firstfile
 $OSTREE fsck -q
 
 echo "ok chmod"
 
+cd ${test_tmpdir}
+rm repo files -rf
+setup_test_repository "bare"
+rev=$($OSTREE rev-parse test2)
+echo -n > repo/objects/${rev:0:2}/${rev:2}.commit
+if $OSTREE fsck -q 2>err.txt; then
+    fatal "fsck unexpectedly succeeded"
+fi
+assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected"
+
+echo "ok metadata checksum"
+
 cd ${test_tmpdir}
 rm repo files -rf
 setup_test_repository "bare"
@@ -42,7 +56,9 @@ rm checkout-test2 -rf
 $OSTREE checkout test2 checkout-test2
 cd checkout-test2
 chmod o+x firstfile
-$OSTREE fsck -q --delete && (echo 1>&2 "fsck unexpectedly succeeded"; exit 1)
+if $OSTREE fsck -q --delete; then
+    fatal "fsck unexpectedly succeeded"
+fi
 
 echo "ok chmod"
 

tests/test-fsck-collections.sh

@@ -21,12 +21,28 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-echo '1..2'
+echo '1..11'
 
 cd ${test_tmpdir}
 
-# Check that fsck detects errors with refs which have collection IDs (i.e. refs in refs/mirrors).
+# Create a new repository with one ref with the repository’s collection ID, and
-set_up_repo() {
+# one ref with a different collection ID (which should be in refs/mirrors).
+set_up_repo_with_collection_id() {
+  rm -rf repo files
+
+  mkdir repo
+  ostree_repo_init repo --collection-id org.example.Collection
+
+  mkdir files
+  pushd files
+  ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 1" -b ref1 > ../ref1-checksum
+  ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 2" --orphan --bind-ref ref2 --add-metadata-string=ostree.collection-binding=org.example.Collection2 > ../ref2-checksum
+  ${CMD_PREFIX} ostree --repo=../repo refs --collections --create=org.example.Collection2:ref2 $(cat ../ref2-checksum)
+  popd
+}
+
+# Create a new repository with one ref and no collection IDs.
+set_up_repo_without_collection_id() {
   rm -rf repo files
 
   mkdir repo
@@ -34,12 +50,12 @@ set_up_repo() {
 
   mkdir files
   pushd files
-  ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 1" -b original-ref > ../original-ref-checksum
+  ${CMD_PREFIX} ostree --repo=../repo commit -s "Commit 3" -b ref3 --bind-ref ref4 > ../ref3-checksum
+  ${CMD_PREFIX} ostree --repo=../repo refs --create=ref4 $(cat ../ref3-checksum)
   popd
-  ${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection:some-ref $(cat original-ref-checksum)
 }
 
-set_up_repo
+set_up_repo_with_collection_id
 
 # fsck at this point should succeed
 ${CMD_PREFIX} ostree fsck --repo=repo > fsck
@@ -47,9 +63,9 @@ assert_file_has_content fsck "^Validating refs in collections...$"
 
 # Drop the commit the ref points to, and drop the original ref so that fsck doesn’t prematurely fail on that.
 find repo/objects -name '*.commit' -delete -print | wc -l > commitcount
-assert_file_has_content commitcount "^1$"
+assert_file_has_content commitcount "^2$"
 
-rm repo/refs/heads/original-ref
+rm repo/refs/heads/ref1
 
 # fsck should now fail
 if ${CMD_PREFIX} ostree fsck --repo=repo > fsck; then
@@ -62,7 +78,7 @@ echo "ok 1 fsck-collections"
 
 # Try fsck in an old repository where refs/mirrors doesn’t exist to begin with.
 # It should succeed.
-set_up_repo
+set_up_repo_with_collection_id
 rm -rf repo/refs/mirrors
 
 ${CMD_PREFIX} ostree fsck --repo=repo > fsck
@@ -70,3 +86,124 @@ assert_file_has_content fsck "^Validating refs...$"
 assert_file_has_content fsck "^Validating refs in collections...$"
 
 echo "ok 2 fsck-collections in old repository"
+
+# Test that fsck detects commits which are pointed to by refs, but which don’t
+# list those refs in their ref-bindings.
+set_up_repo_with_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --create=new-ref $(cat ref1-checksum)
+
+# For compatibility we don't check for this by default
+${CMD_PREFIX} ostree fsck --repo=repo
+# fsck should now fail
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Commit has no requested ref ‘new-ref’ in ref binding metadata (‘ref1’)"
+assert_file_has_content fsck "^Validating refs...$"
+
+echo "ok 3 fsck detects missing ref bindings"
+
+# And the same where the ref is a collection–ref.
+set_up_repo_with_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection2:new-ref $(cat ref1-checksum)
+
+# fsck should now fail
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Commit has no requested ref ‘new-ref’ in ref binding metadata (‘ref1’)"
+assert_file_has_content fsck "^Validating refs...$"
+assert_file_has_content fsck "^Validating refs in collections...$"
+
+echo "ok 4 fsck detects missing collection–ref bindings"
+
+# Check that a ref with a different collection ID but the same ref name is caught.
+set_up_repo_with_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --collections --create=org.example.Collection2:ref1 $(cat ref1-checksum)
+
+# fsck should now fail
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Commit has collection ID ‘org.example.Collection’ in collection binding metadata, while the remote it came from has collection ID ‘org.example.Collection2’"
+assert_file_has_content fsck "^Validating refs...$"
+assert_file_has_content fsck "^Validating refs in collections...$"
+
+echo "ok 5 fsck detects missing collection–ref bindings"
+
+# Check that a commit with ref bindings which aren’t pointed to by refs is OK.
+set_up_repo_with_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --delete ref1
+
+# fsck at this point should succeed
+${CMD_PREFIX} ostree fsck --repo=repo > fsck
+assert_file_has_content fsck "^Validating refs in collections...$"
+
+echo "ok 6 fsck ignores unreferenced ref bindings"
+
+# …but it’s not OK if we pass --verify-back-refs to fsck.
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-back-refs > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Collection–ref (org.example.Collection, ref1) in bindings for commit .* does not exist"
+assert_file_has_content fsck "^Validating refs...$"
+assert_file_has_content fsck "^Validating refs in collections...$"
+
+echo "ok 7 fsck ignores unreferenced ref bindings"
+
+#
+# Now repeat most of the above tests with a repository without collection IDs.
+#
+
+set_up_repo_without_collection_id
+
+# fsck at this point should succeed
+${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck
+assert_file_has_content fsck "^Validating refs in collections...$"
+
+# Drop the commit the ref points to, and drop the original ref so that fsck doesn’t prematurely fail on that.
+find repo/objects -name '*.commit' -delete -print | wc -l > commitcount
+assert_file_has_content commitcount "^1$"
+
+rm repo/refs/heads/ref3
+
+# fsck should now fail
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck; then
+    assert_not_reached "fsck unexpectedly succeeded after deleting commit!"
+fi
+assert_file_has_content fsck "^Validating refs...$"
+
+echo "ok 8 fsck-collections"
+
+# Test that fsck detects commits which are pointed to by refs, but which don’t
+# list those refs in their ref-bindings.
+set_up_repo_without_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --create=new-ref $(cat ref3-checksum)
+
+# fsck should now fail
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-bindings > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Commit has no requested ref ‘new-ref’ in ref binding metadata (‘ref3’, ‘ref4’)"
+assert_file_has_content fsck "^Validating refs...$"
+
+echo "ok 9 fsck detects missing ref bindings"
+
+# Check that a commit with ref bindings which aren’t pointed to by refs is OK.
+set_up_repo_without_collection_id
+${CMD_PREFIX} ostree --repo=repo refs --delete ref3
+
+# fsck at this point should succeed
+${CMD_PREFIX} ostree fsck --repo=repo > fsck
+assert_file_has_content fsck "^Validating refs...$"
+
+echo "ok 10 fsck ignores unreferenced ref bindings"
+
+# …but it’s not OK if we pass --verify-back-refs to fsck.
+if ${CMD_PREFIX} ostree fsck --repo=repo --verify-back-refs > fsck 2> fsck-error; then
+    assert_not_reached "fsck unexpectedly succeeded after adding unbound ref!"
+fi
+assert_file_has_content fsck-error "Ref ‘ref3’ in bindings for commit .* does not exist"
+assert_file_has_content fsck "^Validating refs...$"
+
+echo "ok 11 fsck ignores unreferenced ref bindings"

tests/test-pull-corruption.sh

@@ -76,7 +76,7 @@ if ! skip_one_without_user_xattrs; then
     if ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo fsck 2>err.txt; then
         assert_not_reached "fsck with corrupted commit worked?"
     fi
-    assert_file_has_content err.txt "corrupted object ${corruptrev}\.commit"
+    assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected='${corruptrev}' actual='${rev}'"
 
     # Do a pull-local; this should succeed since we don't verify checksums
     # for local repos by default.

tests/test-refs.sh

@@ -23,7 +23,7 @@ set -euo pipefail
 
 setup_fake_remote_repo1 "archive"
 
-echo '1..5'
+echo '1..6'
 
 cd ${test_tmpdir}
 mkdir repo
@@ -186,3 +186,10 @@ assert_file_has_content_literal refs.txt 'exampleos/x86_64/stable/server -> exam
 ${CMD_PREFIX} ostree --repo=repo summary -u
 
 echo "ok ref symlink"
+
+# https://github.com/ostreedev/ostree/issues/1342
+if ${CMD_PREFIX} ostree --repo=repo refs -A exampleos/x86_64/27/server --create=exampleos:exampleos/x86_64/stable/server 2>err.txt; then
+    fatal "Created alias ref to remote?"
+fi
+assert_file_has_content_literal err.txt 'Cannot create alias to remote ref'
+echo "ok ref no alias remote"

tests/test-remote-add.sh

@@ -21,7 +21,7 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-echo '1..13'
+echo '1..14'
 
 setup_test_repository "bare"
 $OSTREE remote add origin http://example.com/ostree/gnome
@@ -63,6 +63,18 @@ assert_file_has_content list.txt "http://another.com/repo"
 assert_file_has_content list.txt "http://another-noexist.example.com/anotherrepo"
 echo "ok remote list with urls"
 
+cd ${test_tmpdir}
+rm -rf parent-repo
+ostree_repo_init parent-repo
+$OSTREE config set core.parent ${test_tmpdir}/parent-repo
+${CMD_PREFIX} ostree --repo=parent-repo remote add --no-gpg-verify parent-remote http://parent-remote.example.com/parent-remote
+$OSTREE remote list > list.txt
+assert_file_has_content list.txt "origin"
+assert_file_has_content list.txt "another"
+assert_file_has_content list.txt "another-noexist"
+assert_file_has_content list.txt "parent-remote"
+echo "ok remote list with parent repo remotes"
+
 $OSTREE remote delete another
 echo "ok remote delete"
 

tests/test-symbols.sh

@@ -52,7 +52,7 @@ echo 'ok documented symbols'
 
 # ONLY update this checksum in release commits!
 cat > released-sha256.txt <<EOF
-1e2c6b529bc2d940ff8969eaf0377d78d472d41ec2a2fc96e1c79efd7b95d241  ${released_syms}
+3dbe0aa610c7229050f4a651fd18893742f8a24c34f3e25bf807ff98fbfc7f72  ${released_syms}
 EOF
 sha256sum -c released-sha256.txt