diff --git a/debian/changelog b/debian/changelog
index 88eea24e..e3ca3ff6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,8 @@ ostree (2017.10-1) UNRELEASED; urgency=medium
   * Add patches to make the tests pass again when /var/tmp is on tmpfs,
     which does not support user xattrs
   * Add patch to remove useless #! from bash completions
+  * Make build-time test failures fatal again, but only if they are
+    reproducible (at least 3 times out of 5) for now
 
  -- Simon McVittie <smcv@debian.org>  Tue, 29 Aug 2017 16:08:09 +0100
 
diff --git a/debian/test.sh b/debian/test.sh
index 4a8f2b6e..97c54646 100755
--- a/debian/test.sh
+++ b/debian/test.sh
@@ -29,7 +29,10 @@ fi
 # There are several race conditions that cause intermittent failures.
 # They are not actually a regression - we've just been luckier in the
 # past - so let newer versions build reliably.
-if [ "$failed" -gt 0 ]; then
+if [ "$failed" -gt 2 ]; then
+    echo "Failed $failed out of $try_tests test runs; that seems bad"
+    exit 1
+elif [ "$failed" -gt 0 ]; then
     echo "Failed $failed out of $try_tests test runs; continuing anyway"
 else
     echo "All tests passed"