New upstream version 2017.1

tests/admin-test.sh

@@ -1,4 +1,5 @@
-#!/bin/bash
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011,2014 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or

tests/archive-test.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or

tests/basic-test.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or
@@ -26,7 +26,7 @@ echo "ok checkout"
 
 $OSTREE rev-parse test2
 $OSTREE rev-parse 'test2^'
-$OSTREE rev-parse 'test2^^' 2>/dev/null && (echo 1>&2 "rev-parse test2^^ unexpectedly succeeded!"; exit 1)
+$OSTREE rev-parse 'test2^^' 2>/dev/null && fatal "rev-parse test2^^ unexpectedly succeeded!"
 echo "ok rev-parse"
 
 checksum=$($OSTREE rev-parse test2)
@@ -294,7 +294,7 @@ rm repo3/refs/heads/* repo3/refs/remotes/* -rf
 ${CMD_PREFIX} ostree --repo=repo3 prune --refs-only
 find repo3/objects -name '*.commit' > objlist-after-prune
 if cmp -s objlist-before-prune objlist-after-prune; then
-    echo "Prune didn't delete anything!"; exit 1
+    fatal "Prune didn't delete anything!"
 fi
 rm repo3 objlist-before-prune objlist-after-prune -rf
 echo "ok prune"

tests/libtest.sh

@@ -29,9 +29,13 @@ else
   test_builddir=$(dirname $0)
 fi
 
-assert_not_reached () {
+fatal() {
     echo $@ 1>&2; exit 1
 }
+# fatal() is shorter to type, but retain this alias
+assert_not_reached () {
+    fatal "$@"
+}
 
 test_tmpdir=$(pwd)
 
@@ -102,55 +106,58 @@ else
     fi
 fi
 
+if test -n "${OSTREE_UNINSTALLED:-}"; then
+    OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd
+else
+    OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd"
+fi
+
 assert_streq () {
-    test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1)
+    test "$1" = "$2" || fatal "$1 != $2"
 }
 
 assert_str_match () {
     if ! echo "$1" | grep -E -q "$2"; then
-	(echo 1>&2 "$1 does not match regexp $2"; exit 1)
+	      fatal "$1 does not match regexp $2"
     fi
 }
 
 assert_not_streq () {
-    (! test "$1" = "$2") || (echo 1>&2 "$1 == $2"; exit 1)
+    (! test "$1" = "$2") || fatal "$1 == $2"
 }
 
 assert_has_file () {
-    test -f "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1)
+    test -f "$1" || fatal "Couldn't find '$1'"
 }
 
 assert_has_dir () {
-    test -d "$1" || (echo 1>&2 "Couldn't find '$1'"; exit 1)
+    test -d "$1" || fatal "Couldn't find '$1'"
 }
 
 assert_not_has_file () {
     if test -f "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' exists"
-        exit 1
+        fatal "File '$1' exists"
     fi
 }
 
 assert_not_file_has_content () {
     if grep -q -e "$2" "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' incorrectly matches regexp '$2'"
-        exit 1
+        fatal "File '$1' incorrectly matches regexp '$2'"
     fi
 }
 
 assert_not_has_dir () {
     if test -d "$1"; then
-	echo 1>&2 "Directory '$1' exists"; exit 1
+	      fatal "Directory '$1' exists"
     fi
 }
 
 assert_file_has_content () {
     if ! grep -q -e "$2" "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' doesn't match regexp '$2'"
-        exit 1
+        fatal "File '$1' doesn't match regexp '$2'"
     fi
 }
 
@@ -169,8 +176,7 @@ assert_symlink_has_content () {
 assert_file_empty() {
     if test -s "$1"; then
         sed -e 's/^/# /' < "$1" >&2
-        echo 1>&2 "File '$1' is not empty"
-        exit 1
+        fatal "File '$1' is not empty"
     fi
 }
 
@@ -178,8 +184,7 @@ assert_files_hardlinked() {
     f1=$(stat -c %i $1)
     f2=$(stat -c %i $2)
     if [ "$f1" != "$f2" ]; then
-        echo 1>&2 "Files '$1' and '$2' are not hardlinked"
-        exit 1
+        fatal "Files '$1' and '$2' are not hardlinked"
     fi
 }
 
@@ -257,7 +262,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 
@@ -379,7 +384,7 @@ EOF
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir} ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port
+    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 

tests/pull-test.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
-#
+# This file is to be sourced, not executed
+
 # Copyright (C) 2011 Colin Walters <walters@verbum.org>
 #
 # This library is free software; you can redistribute it and/or

tests/test-basic-user.sh

@@ -23,9 +23,6 @@ set -euo pipefail
 
 skip_without_user_xattrs
 
-echo "1..1"
-
 setup_test_repository "bare-user"
-echo "ok setup"
 
 . $(dirname $0)/basic-test.sh

tests/test-commit-sign.sh

@@ -53,7 +53,7 @@ cd ${test_tmpdir}
 mkdir ${test_tmpdir}/httpd
 cd httpd
 ln -s ${test_tmpdir}/ostree-srv ostree
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
 port=$(cat ${test_tmpdir}/httpd-port)
 assert_streq $port 18081
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address

tests/test-prune.sh

@@ -25,7 +25,7 @@ skip_without_user_xattrs
 
 setup_fake_remote_repo1 "archive-z2"
 
-echo '1..3'
+echo '1..5'
 
 cd ${test_tmpdir}
 mkdir repo
@@ -49,6 +49,7 @@ find repo | grep \.commit$ | wc -l > commitcount
 assert_file_has_content commitcount "^3$"
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_file_has_content tombstonecommitcount "^0$"
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=1 -v
 find repo | grep \.commit$ | wc -l > commitcount
@@ -73,6 +74,7 @@ find repo/objects -name '*.commit' | wc -l > commitcount
 assert_file_has_content commitcount "^1$"
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_not_file_has_content tombstonecommitcount "^0$"
+$OSTREE fsck
 
 # and that tombstone are deleted once the commits are pulled again
 ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test
@@ -83,6 +85,7 @@ COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo log test | grep ^commit | cu
 ${CMD_PREFIX} ostree --repo=repo prune --delete-commit=$COMMIT_TO_DELETE
 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount
 assert_file_has_content tombstonecommitcount "^1$"
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v
 find repo/objects -name '*.commit' | wc -l > commitcount
@@ -94,6 +97,7 @@ assert_file_has_content commitcount "^3$"
 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="2015-10-29 12:43:29 +0000"
 find repo/objects -name '*.commit' | wc -l > commitcount
 assert_file_has_content commitcount "^2$"
+$OSTREE fsck
 
 
 ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v
@@ -112,6 +116,7 @@ oldcommit_rev=$($OSTREE --repo=repo rev-parse oldcommit)
 $OSTREE ls ${oldcommit_rev}
 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago"
 $OSTREE ls ${oldcommit_rev}
+$OSTREE fsck
 
 ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test
 ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="November 05 1955"
@@ -124,6 +129,7 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount
 assert_file_has_content deltascount "^2$"
 COMMIT_TO_DELETE=$(${CMD_PREFIX} ostree --repo=repo rev-parse test)
 ${CMD_PREFIX} ostree --repo=repo prune --static-deltas-only --delete-commit=$COMMIT_TO_DELETE
+${CMD_PREFIX} ostree --repo=repo fsck
 ${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount
 assert_file_has_content deltascount "^1$"
 ${CMD_PREFIX} ostree --repo=repo static-delta generate test
@@ -178,3 +184,62 @@ ${CMD_PREFIX} ostree --repo=child-repo prune --refs-only --depth=0
 assert_has_n_objects child-repo 3
 
 echo "ok prune with parent repo"
+
+# Delete all the above since I can't be bothered to think about how new tests
+# would interact. We make a new repo test suite, then clone it
+# for "subtests" below with reinitialize_datesnap_repo()
+rm repo datetest-snapshot-repo -rf
+${CMD_PREFIX} ostree --repo=datetest-snapshot-repo init --mode=archive
+# Some ancient commits on the both a stable/dev branch
+for day in $(seq 5); do
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=stable -m test -s "old stable build $day" tree --timestamp="October $day 1985"
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=dev -m test -s "old dev build $day" tree --timestamp="October $day 1985"
+done
+# And some new ones
+for x in $(seq 3); do
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=stable -m test -s "new stable build $x" tree
+    ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=dev -m test -s "new dev build $x" tree
+done
+find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^16$"
+
+# Snapshot the above
+reinitialize_datesnap_repo() {
+    rm repo -rf
+    ${CMD_PREFIX} ostree --repo=repo init --mode=archive
+    ${CMD_PREFIX} ostree --repo=repo pull-local --depth=-1 datetest-snapshot-repo 
+}
+
+# This test prunes with both younger than as well as a full strong ref to the
+# stable branch
+reinitialize_datesnap_repo
+# First, a quick test of invalid input
+if ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=BACON 2>err.txt; then
+    assert_not_reached "BACON is a number?!"
+fi
+assert_file_has_content err.txt 'Invalid depth BACON'
+${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=-1
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^11$"
+# Double check our backup is unchanged
+find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^16$"
+$OSTREE fsck
+
+# Again but this time only retain 6 (5+1) commits on stable.  This should drop
+# out 8 - 6 = 2 commits (so the 11 above minus 2 = 9)
+${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=5
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^9$"
+$OSTREE fsck
+echo "ok retain branch depth and keep-younger-than"
+
+# Just stable branch ref, we should prune everything except the tip of dev,
+# so 8 stable + 1 dev = 9
+reinitialize_datesnap_repo
+${CMD_PREFIX} ostree --repo=repo prune --depth=0 --retain-branch-depth=stable=-1
+find repo/objects -name '*.commit' | wc -l > commitcount
+assert_file_has_content commitcount "^9$"
+$OSTREE fsck
+
+echo "ok retain branch depth (alone)"

tests/test-pull-c.c

@@ -49,6 +49,8 @@ test_data_init (TestData *td)
   if (!g_file_get_contents ("httpd-address", &http_address, NULL, error))
     goto out;
 
+  g_strstrip (http_address);
+
   repo_url = g_strconcat (http_address, "/ostree/gnomerepo", NULL);
 
   { g_autoptr(GVariantBuilder) builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));

tests/test-pull-contenturl.sh

@@ -51,8 +51,7 @@ fi
 find ${test_tmpdir}/ostree-srv/gnomerepo/objects \
   ! -name '*.commitmeta' -type f | xargs rm
 
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
-  -p ${test_tmpdir}/httpd-content-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-content-port
 content_port=$(cat ${test_tmpdir}/httpd-content-port)
 echo "http://127.0.0.1:${content_port}" > ${test_tmpdir}/httpd-content-address
 

tests/test-pull-many.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+cd ${test_tmpdir}
+rm ostree-srv/gnomerepo/ -rf
+mkdir ostree-srv/gnomerepo/
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo init --mode=archive
+
+echo '1..1'
+
+# Simulate a large archive pull from scratch.  Large here is
+# something like the Fedora Atomic Workstation branch which has
+# objects: meta: 7497 content: 103541
+# 9443 directories, 7097 symlinks, 112832 regfiles
+# So we'll make ~11 files per dir, with one of them a symlink
+
+cd ${test_tmpdir}
+rm main -rf
+mkdir main
+cd main
+ndirs=9443
+depth=0
+echo "$(date): Generating content..."
+set +x  # No need to spam the logs for this
+while [ $ndirs -gt 0 ]; do
+    # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up
+    x=$(($ndirs % 3))
+    case $x in
+        0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;;
+        1|2) if [ $depth -lt 9 ]; then
+                 mkdir dir-${ndirs}
+                 cd dir-${ndirs}
+                 depth=$((depth+1))
+             else
+                 if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi
+             fi ;;
+    esac
+
+    # One symlink - we use somewhat predictable content to have dupes
+    ln -s $(($x % 20)) link-$ndirs
+    # 10 files
+    nfiles=10
+    while [ $nfiles -gt 0 ]; do
+        echo file-$ndirs-$nfiles > f$ndirs-$nfiles
+        nfiles=$((nfiles-1))
+    done
+    ndirs=$((ndirs-1))
+done
+set -x
+cd ${test_tmpdir}
+mkdir build-repo
+${CMD_PREFIX} ostree --repo=build-repo init --mode=bare-user
+echo "$(date): Committing content..."
+${CMD_PREFIX} ostree --repo=build-repo commit -b main -s 'big!' --tree=dir=main
+for x in commit dirtree dirmeta file; do
+    find build-repo/objects -name '*.'${x} |wc -l > ${x}count
+    echo "$x: " $(cat ${x}count)
+done
+assert_file_has_content commitcount '^1$'
+assert_file_has_content dirmetacount '^1$'
+assert_file_has_content filecount '^94433$'
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo pull-local build-repo
+
+echo "$(date): Pulling content..."
+rm repo -rf
+${CMD_PREFIX} ostree --repo=repo init --mode=archive
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+${CMD_PREFIX} ostree --repo=repo pull --mirror origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+for x in commit dirtree dirmeta filez; do
+    find repo/objects -name '*.'${x} |wc -l > ${x}count
+    echo "$x: " $(cat ${x}count)
+done
+assert_file_has_content commitcount '^1$'
+assert_file_has_content dirmetacount '^1$'
+assert_file_has_content filezcount '^94433$'
+
+echo "ok"

tests/test-pull-metalink.sh

@@ -29,7 +29,7 @@ echo '1..9'
 cd ${test_tmpdir}
 mkdir metalink-data
 cd metalink-data
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
 metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port)
 echo "http://127.0.0.1:${metalink_port}" > ${test_tmpdir}/metalink-httpd-address
 

tests/test-pull-mirrorlist.sh

@@ -33,7 +33,7 @@ setup_mirror () {
   cd $name
   cp -a ${test_tmpdir}/ostree-srv ostree
 
-  ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
+  ${OSTREE_HTTPD} --autoexit --daemonize \
     -p ${test_tmpdir}/${name}-port
   port=$(cat ${test_tmpdir}/${name}-port)
   echo "http://127.0.0.1:${port}" > ${test_tmpdir}/${name}-address

tests/test-pull-override-url.sh

@@ -50,7 +50,7 @@ mkdir mirror-httpd
 cd mirror-httpd
 ln -s ${test_tmpdir}/mirror-srv ostree
 mirror_log="${test_tmpdir}/mirror_log"
-${CMD_PREFIX} ostree trivial-httpd --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
+${OSTREE_HTTPD} --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
 port=$(cat ${test_tmpdir}/mirror-httpd-port)
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/mirror-httpd-address
 

tests/test-pull-repeated.sh

@@ -34,7 +34,7 @@ for x in $(seq 200); do
 	echo "Success on iteration ${x}"
 	break;
     fi
-    assert_file_has_content err.txt "500.*Internal Server Error"
+    assert_file_has_content err.txt "\(500.*Internal Server Error\)\|\(HTTP 500\)"
 done
 
 ${CMD_PREFIX} ostree --repo=repo fsck

tests/test-pull-resume.sh

@@ -42,7 +42,7 @@ do
   if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.log; then
     break
   fi
-  assert_file_has_content err.log 'error:.*Download incomplete'
+  assert_file_has_content err.log 'error:.*\(Download incomplete\)\|\(Transferred a partial file\)'
 done
 if ${CMD_PREFIX} ostree --repo=repo fsck; then
     echo "ok, pull succeeded!"