New upstream version 2017.13

tests/admin-test.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..$((22 + ${extra_admin_tests:-0}))"
+echo "1..$((23 + ${extra_admin_tests:-0}))"
 
 function validate_bootloader() {
     cd ${test_tmpdir};
@@ -277,6 +277,14 @@ assert_streq ${curr_rev} ${head_rev}
 
 echo "ok upgrade with and without override-commit"
 
+${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit  --add-metadata-string "version=${version}" \
+              --add-metadata-string 'ostree.source-title=libtest os_repository_new_commit()' -b testos/buildmaster/x86_64-runtime \
+              -s "Build" --tree=dir=${test_tmpdir}/osdata
+${CMD_PREFIX} ostree admin upgrade --os=testos
+${CMD_PREFIX} ostree admin status | tee status.txt
+assert_file_has_content_literal status.txt '`- libtest os_repository_new_commit()'
+echo "ok source title"
+
 deployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir)
 ${CMD_PREFIX} ostree --sysroot=sysroot remote add --set=gpg-verify=false remote-test-physical file://$(pwd)/testos-repo
 assert_not_has_file ${deployment}/etc/ostree/remotes.d/remote-test-physical.conf testos-repo

tests/basic-test.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..$((73 + ${extra_basic_tests:-0}))"
+echo "1..$((77 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -177,6 +177,43 @@ assert_file_has_content yet/another/tree/green 'leaf'
 assert_file_has_content four '4'
 echo "ok cwd contents"
 
+cd ${test_tmpdir}
+rm checkout-test2-l -rf
+$OSTREE checkout ${CHECKOUT_H_ARGS} test2 $test_tmpdir/checkout-test2-l
+date > $test_tmpdir/checkout-test2-l/newdatefile.txt
+$OSTREE commit ${COMMIT_ARGS} --link-checkout-speedup --consume -b test2 --tree=dir=$test_tmpdir/checkout-test2-l
+assert_not_has_dir $test_tmpdir/checkout-test2-l
+$OSTREE fsck
+# Some of the later tests are sensitive to state
+$OSTREE reset test2 test2^
+$OSTREE prune --refs-only
+echo "ok consume (nom nom nom)"
+
+# Test adopt
+cd ${test_tmpdir}
+rm checkout-test2-l -rf
+$OSTREE checkout ${CHECKOUT_H_ARGS} test2 $test_tmpdir/checkout-test2-l
+echo 'a file to consume 🍔' > $test_tmpdir/checkout-test2-l/eatme.txt
+# Save a link to it for device/inode comparison
+ln $test_tmpdir/checkout-test2-l/eatme.txt $test_tmpdir/eatme-savedlink.txt
+$OSTREE commit ${COMMIT_ARGS} --link-checkout-speedup --consume -b test2 --tree=dir=$test_tmpdir/checkout-test2-l
+$OSTREE fsck
+# Adoption isn't implemented for bare-user yet
+eatme_objpath=$(ostree_file_path_to_object_path repo test2 /eatme.txt)
+if grep -q '^mode=bare$' repo/config || is_bare_user_only_repo repo; then
+    assert_files_hardlinked ${test_tmpdir}/eatme-savedlink.txt ${eatme_objpath}
+else
+    if files_are_hardlinked ${test_tmpdir}/eatme-savedlink.txt ${eatme_objpath}; then
+        fatal "bare-user adopted?"
+    fi
+fi
+assert_not_has_dir $test_tmpdir/checkout-test2-l
+# Some of the later tests are sensitive to state
+$OSTREE reset test2 test2^
+$OSTREE prune --refs-only
+rm -f ${test_tmpdir}/eatme-savedlink.txt
+echo "ok adopt"
+
 cd ${test_tmpdir}
 $OSTREE commit ${COMMIT_ARGS} -b test2-no-parent -s '' $test_tmpdir/checkout-test2-4
 assert_streq $($OSTREE log test2-no-parent |grep '^commit' | wc -l) "1"
@@ -228,6 +265,33 @@ fi
 assert_file_has_content err.txt "No such metadata object"
 echo "ok commit orphaned"
 
+cd ${test_tmpdir}
+# in bare-user-only mode, we canonicalize ownership to 0:0, so checksums won't
+# match -- we could add a --ignore-ownership option I suppose?
+if is_bare_user_only_repo repo; then
+    echo "ok # SKIP checksums won't match up in bare-user-only"
+else
+    $OSTREE fsck
+    CHECKSUM_FLAG=
+    if [ -n "${OSTREE_NO_XATTRS:-}" ]; then
+        CHECKSUM_FLAG=--ignore-xattrs
+    fi
+    rm -rf checksum-test
+    $OSTREE checkout test2 checksum-test
+    find checksum-test/ -type f | while read fn; do
+        checksum=$($CMD_PREFIX ostree checksum $CHECKSUM_FLAG $fn)
+        objpath=repo/objects/${checksum::2}/${checksum:2}.file
+        assert_has_file $objpath
+        # running `ostree checksum` on the obj might not necessarily match, let's
+        # just check that they have the same content to confirm that it's
+        # (probably) the originating file
+        object_content_checksum=$(sha256sum $objpath | cut -f1 -d' ')
+        checkout_content_checksum=$(sha256sum $fn | cut -f1 -d' ')
+        assert_streq "$object_content_checksum" "$checkout_content_checksum"
+    done
+    echo "ok checksum CLI"
+fi
+
 cd ${test_tmpdir}
 $OSTREE diff test2^ test2 > diff-test2
 assert_file_has_content diff-test2 'D */a/5'
@@ -334,6 +398,14 @@ echo "ok commit from ref with modifier"
 $OSTREE commit ${COMMIT_ARGS} -b trees/test2 -s 'ref with / in it' --tree=ref=test2
 echo "ok commit ref with /"
 
+mkdir badutf8
+echo "invalid utf8 filename" > badutf8/$(printf '\x80')
+if $OSTREE commit ${COMMIT_ARGS} -b badutf8 --tree=dir=badutf8 2>err.txt; then
+    assert_not_reached "commit filename with invalid UTF-8"
+fi
+assert_file_has_content err.txt "Invalid UTF-8 in filename"
+echo "ok commit bad UTF-8"
+
 old_rev=$($OSTREE rev-parse test2)
 $OSTREE ls -R -C test2
 $OSTREE commit ${COMMIT_ARGS} --skip-if-unchanged -b trees/test2 -s 'should not be committed' --tree=ref=test2
@@ -501,6 +573,12 @@ for x in $(seq 3); do
         assert_file_has_content union-identical-test/usr/share/licenses/${v} GPL
     done
 done
+# now checkout the first pkg in force copy mode to make sure we can checksum
+rm union-identical-test -rf
+$OSTREE checkout --force-copy union-identical-pkg1 union-identical-test
+for x in 2 3; do
+    $OSTREE checkout ${CHECKOUT_H_ARGS} --union-identical union-identical-pkg${x} union-identical-test
+done
 echo "ok checkout union identical merges"
 
 # Make conflicting packages, one with regfile, one with symlink
@@ -801,6 +879,20 @@ if touch overlay/baz/.wh.cow && touch overlay/.wh.deeper; then
     assert_has_file overlay-co/anewdir/blah
     assert_has_file overlay-co/anewfile
 
+    # And test replacing a directory wholesale with a symlink as well as a regular file
+    mkdir overlay
+    echo baz to file > overlay/baz
+    ln -s anewfile overlay/anewdir
+    $OSTREE --repo=repo commit ${COMMIT_ARGS} -b overlay-dir-convert --tree=dir=overlay
+    rm overlay -rf
+
+    rm overlay-co -rf
+    for branch in test2 overlay-dir-convert; do
+        $OSTREE --repo=repo checkout --union --whiteouts ${branch} overlay-co
+    done
+    assert_has_file overlay-co/baz
+    test -L overlay-co/anewdir
+
     echo "ok whiteouts enabled"
 
     # Now double check whiteouts are not processed without --whiteouts

tests/libostreetest.c

@@ -85,8 +85,10 @@ ot_check_relabeling (gboolean *can_relabel,
   g_autoptr(GBytes) bytes = glnx_fgetxattr_bytes (tmpf.fd, "security.selinux", &local_error);
   if (!bytes)
     {
-      /* libglnx preserves errno */
-      if (G_IN_SET (errno, ENOTSUP, ENODATA))
+      /* libglnx preserves errno. The EOPNOTSUPP case can't be part of a
+       * 'case' statement because on most but not all architectures,
+       * it's numerically equal to ENOTSUP. */
+      if (G_IN_SET (errno, ENOTSUP, ENODATA) || errno == EOPNOTSUPP)
         {
           *can_relabel = FALSE;
           return TRUE;
@@ -99,7 +101,7 @@ ot_check_relabeling (gboolean *can_relabel,
   const guint8 *data = g_bytes_get_data (bytes, &data_len);
   if (fsetxattr (tmpf.fd, "security.selinux", data, data_len, 0) < 0)
     {
-      if (errno == ENOTSUP)
+      if (errno == ENOTSUP || errno == EOPNOTSUPP)
         {
           *can_relabel = FALSE;
           return TRUE;
@@ -122,7 +124,7 @@ ot_check_user_xattrs (gboolean *has_user_xattrs,
 
   if (fsetxattr (tmpf.fd, "user.test", "novalue", strlen ("novalue"), 0) < 0)
     {
-      if (errno == ENOTSUP)
+      if (errno == ENOTSUP || errno == EOPNOTSUPP)
         {
           *has_user_xattrs = FALSE;
           return TRUE;

tests/libtest.sh

@@ -229,6 +229,8 @@ ostree_repo_init() {
     fi
 }
 
+# The original one; use setup_fake_remote_repo2 for newer code,
+# down the line we'll try to port tests.
 setup_fake_remote_repo1() {
     mode=$1
     commit_opts=${2:-}
@@ -267,103 +269,47 @@ setup_fake_remote_repo1() {
     export OSTREE="${CMD_PREFIX} ostree --repo=repo"
 }
 
-# Set up a large repository for stress testing.
-# Something like the Fedora Atomic Workstation branch which has
-# objects: meta: 7497 content: 103541
-# 9443 directories, 7097 symlinks, 112832 regfiles
-# So we'll make ~11 files per dir, with one of them a symlink
-# Actually, let's cut this down to 1/3 which is still useful.  So:
-# 3147 dirs, with still ~11 files per dir, for 37610 content objects
-setup_exampleos_repo() {
-    args=${1:-}
-    cd ${test_tmpdir}
+# Newer version of the above with more "real" data
+setup_fake_remote_repo2() {
+    mode=$1
+    commit_opts=${2:-}
+    args=${3:-}
+    shift
+    oldpwd=`pwd`
     mkdir ostree-srv
-    mkdir -p ostree-srv/exampleos/{repo,build-repo}
-    export ORIGIN_REPO=ostree-srv/exampleos/repo
-    export ORIGIN_BUILD_REPO=ostree-srv/exampleos/build-repo
-    ostree_repo_init ${ORIGIN_REPO} --mode=archive
-    ostree_repo_init ${ORIGIN_BUILD_REPO} --mode=bare-user
+    cd ostree-srv
+    mkdir repo
+    ostree_repo_init repo --mode=$mode
+    # Backcompat
+    ln -sr repo gnomerepo
+    # Initialize content
+    mkdir files
+    cd files
+    mkdir -p usr/{etc,bin,lib,share}
+    ln -sr usr/bin bin
+    ln -sr usr/lib lib
+    tar xf ${test_srcdir}/fah-deltadata-old.tar.xz
+    remote_ref=exampleos/42/x86_64/main
+    cd ..
+    ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/repo commit \
+                  --consume $commit_opts --add-metadata-string version=42.0 -b ${remote_ref} \
+                  --tree=dir=files
+    test '!' -d files
+    ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/repo checkout -U ${remote_ref} files
+    (cd files && tar xf ${test_srcdir}/fah-deltadata-new.tar.xz)
+    ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/repo commit \
+                  --consume $commit_opts --add-metadata-string version=42.1 -b ${remote_ref} \
+                  --tree=dir=files
+
+    # And serve via HTTP
     cd ${test_tmpdir}
-    rm main -rf
-    mkdir main
-    cd main
-    ndirs=3147
-    depth=0
-    set +x  # No need to spam the logs for this
-    echo "$(date): Generating initial content..."
-    while [ $ndirs -gt 0 ]; do
-        # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up
-        x=$(($ndirs % 3))
-        case $x in
-            0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;;
-            1|2) if [ $depth -lt 9 ]; then
-                         mkdir dir-${ndirs}
-                         cd dir-${ndirs}
-                         depth=$((depth+1))
-                     else
-                         if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi
-                 fi ;;
-        esac
-        # One symlink - we use somewhat predictable content to have dupes
-        ln -s $(($x % 20)) link-$ndirs
-        # 10 files
-        nfiles=10
-        while [ $nfiles -gt 0 ]; do
-            echo file-$ndirs-$nfiles > f$ndirs-$nfiles
-            # Make an unreadable file to trigger https://github.com/ostreedev/ostree/pull/634
-            if [ $(($x % 10)) -eq 0 ]; then
-                chmod 0600 f$ndirs-$nfiles
-            fi
-            nfiles=$((nfiles-1))
-        done
-        ndirs=$((ndirs-1))
-    done
-    cd ${test_tmpdir}
-    set -x
-
-    export REF=exampleos/42/standard
-
-    ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit -b ${REF} --tree=dir=main
-    rm main -rf
-    ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} checkout ${REF} main
-
-    find main > files.txt
-    nfiles=$(wc -l files.txt | cut -f 1 -d ' ')
-    # We'll make 5 more commits
-    for iter in $(seq 5); do
-        set +x
-        # Change 10% of files
-        for fiter in $(seq $(($nfiles / 10))); do
-            filenum=$(($RANDOM % ${nfiles}))
-            set +o pipefail
-            filename=$(tail -n +${filenum} < files.txt | head -1)
-            set -o pipefail
-            if test -f $filename; then
-                rm -f $filename
-                echo file-${iter}-${fiter} > ${filename}
-            fi
-        done
-        set -x
-        ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit --link-checkout-speedup -b ${REF} --tree=dir=main
-    done
-
-    ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} pull-local --depth=-1 ${ORIGIN_BUILD_REPO}
-
-    for x in "^^" "^" ""; do
-        ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} static-delta generate --from="${REF}${x}^" --to="${REF}${x}"
-    done
-    ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} summary -u
-
-    cd ${test_tmpdir}/ostree-srv
-    mkdir httpd
-    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd/httpd.log --daemonize -p httpd/port $args
-    port=$(cat httpd/port)
-    echo "http://127.0.0.1:${port}" > httpd/address
-
-    cd ${test_tmpdir}
-    rm repo -rf
-    ostree_repo_init repo --mode=bare-user
-    ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat ostree-srv/httpd/address)/exampleos/repo
+    mkdir ${test_tmpdir}/httpd
+    cd httpd
+    ln -s ${test_tmpdir}/ostree-srv ostree
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args
+    port=$(cat ${test_tmpdir}/httpd-port)
+    echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
+    cd ${oldpwd}
     export OSTREE="${CMD_PREFIX} ostree --repo=repo"
 }
 

tests/pull-test.sh

@@ -27,15 +27,32 @@ function repo_init() {
     ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@"
 }
 
+repo_init --no-gpg-verify
+
+# See also the copy of this in basic-test.sh
+COMMIT_ARGS=""
+CHECKOUT_U_ARG=""
+CHECKOUT_H_ARGS="-H"
+if is_bare_user_only_repo repo; then
+    COMMIT_ARGS="--canonical-permissions"
+    # Also, since we can't check out uid=0 files we need to check out in user mode
+    CHECKOUT_U_ARG="-U"
+    CHECKOUT_H_ARGS="-U -H"
+else
+    if grep -E -q '^mode=bare-user' repo/config; then
+        CHECKOUT_H_ARGS="-U -H"
+    fi
+fi
+
 function verify_initial_contents() {
     rm checkout-origin-main -rf
-    $OSTREE checkout origin/main checkout-origin-main
+    $OSTREE checkout ${CHECKOUT_H_ARGS} origin/main checkout-origin-main
     cd checkout-origin-main
     assert_file_has_content firstfile '^first$'
     assert_file_has_content baz/cow '^moo$'
 }
 
-echo "1..31"
+echo "1..33"
 
 # Try both syntaxes
 repo_init --no-gpg-verify
@@ -61,7 +78,7 @@ echo "ok pull mirror"
 
 mkdir otherbranch
 echo someothercontent > otherbranch/someothercontent
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b otherbranch --tree=dir=otherbranch
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b otherbranch --tree=dir=otherbranch
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 rm mirrorrepo -rf
 # All refs
@@ -88,9 +105,9 @@ echo "ok pull mirror (ref subset with summary)"
 
 cd ${test_tmpdir}
 rm checkout-origin-main -rf
-$OSTREE --repo=ostree-srv/gnomerepo checkout main checkout-origin-main
+$OSTREE --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main checkout-origin-main
 echo moomoo > checkout-origin-main/baz/cow
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main -s "" --tree=dir=checkout-origin-main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s "" --tree=dir=checkout-origin-main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo fsck
@@ -115,11 +132,12 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo pull --bareuseronly-files origin main
 echo "ok pull (bareuseronly, safe)"
 
 rm checkout-origin-main -rf
-$OSTREE --repo=ostree-srv/gnomerepo checkout main checkout-origin-main
+$OSTREE --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main checkout-origin-main
 cat > statoverride.txt <<EOF
 2048 /some-setuid
 EOF
 echo asetuid > checkout-origin-main/some-setuid
+# Don't use ${COMMIT_ARGS} as we don't want --canonical-permissions with bare-user-only
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b content-with-suid --statoverride=statoverride.txt --tree=dir=checkout-origin-main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 # Verify we reject it both when unpacking and when mirroring
@@ -140,7 +158,8 @@ echo "ok pull (bareuseronly mirror)"
 # Corruption tests <https://github.com/ostreedev/ostree/issues/1211>
 cd ${test_tmpdir}
 repo_init --no-gpg-verify
-if ! is_bare_user_only_repo repo && ! skip_one_without_user_xattrs; then
+if ! is_bare_user_only_repo repo; then
+if ! skip_one_without_user_xattrs; then
     if is_bare_user_only_repo repo; then
         cacherepomode=bare-user-only
     else
@@ -193,15 +212,19 @@ if ! is_bare_user_only_repo repo && ! skip_one_without_user_xattrs; then
     repo_init --no-gpg-verify
     echo "ok corruption"
 fi
+else
+# bareuseronly case, we don't mark it as SKIP at the moment
+echo "ok corruption (skipped)"
+fi
 
 cd ${test_tmpdir}
 rm mirrorrepo/refs/remotes/* -rf
 ${CMD_PREFIX} ostree --repo=mirrorrepo prune --refs-only
 ${CMD_PREFIX} ostree --repo=mirrorrepo pull origin main
 rm checkout-origin-main -rf
-$OSTREE --repo=ostree-srv/gnomerepo checkout main checkout-origin-main
+$OSTREE --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main checkout-origin-main
 echo yetmorecontent > checkout-origin-main/baz/cowtest
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main -s "" --tree=dir=checkout-origin-main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s "" --tree=dir=checkout-origin-main
 rev=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo rev-parse main)
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
@@ -223,8 +246,8 @@ cd ${test_tmpdir}
 rm otherrepo -rf
 ostree_repo_init otherrepo --mode=archive
 rm checkout-origin-main -rf
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main checkout-origin-main
-${CMD_PREFIX} ostree --repo=otherrepo commit -b localbranch --tree=dir=checkout-origin-main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main checkout-origin-main
+${CMD_PREFIX} ostree --repo=otherrepo commit ${COMMIT_ARGS} -b localbranch --tree=dir=checkout-origin-main
 ${CMD_PREFIX} ostree --repo=otherrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo
 ${CMD_PREFIX} ostree --repo=otherrepo pull origin main
 rm mirrorrepo-local -rf
@@ -238,8 +261,16 @@ ${CMD_PREFIX} ostree --repo=mirrorrepo-local rev-parse localbranch
 ${CMD_PREFIX} ostree --repo=mirrorrepo-local fsck
 echo "ok pull-local mirror errors with mixed refs"
 
+rm -f otherrepo/summary
+if ${CMD_PREFIX} ostree --repo=mirrorrepo-local pull-local otherrepo nosuchbranch 2>err.txt; then
+    fatal "pulled nonexistent branch"
+fi
+# So true
+assert_file_has_content_literal err.txt "error: Refspec 'nosuchbranch' not found"
+echo "ok pull-local nonexistent branch"
+
 cd ${test_tmpdir}
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main -s "Metadata string" --add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s "Metadata string" --add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 ${CMD_PREFIX} ostree --repo=repo pull origin main
 ${CMD_PREFIX} ostree --repo=repo fsck
@@ -270,13 +301,13 @@ origrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse main)
 # Check we can pull the same commit with timestamp checking enabled
 ${CMD_PREFIX} ostree --repo=repo pull -T origin main
 assert_streq ${origrev} "$(${CMD_PREFIX} ostree --repo=repo rev-parse main)"
-newrev=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit -b main --tree=ref=main)
+newrev=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main --tree=ref=main)
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 # New commit with timestamp checking
 ${CMD_PREFIX} ostree --repo=repo pull -T origin main
 assert_not_streq "${origrev}" "${newrev}"
 assert_streq ${newrev} "$(${CMD_PREFIX} ostree --repo=repo rev-parse main)"
-newrev2=$(${CMD_PREFIX} ostree --timestamp="October 25 1985" --repo=ostree-srv/gnomerepo commit -b main --tree=ref=main)
+newrev2=$(${CMD_PREFIX} ostree --timestamp="October 25 1985" --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main --tree=ref=main)
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
 if ${CMD_PREFIX} ostree --repo=repo pull -T origin main 2>err.txt; then
     fatal "pulled older commit with timestamp checking enabled?"
@@ -296,12 +327,12 @@ ${CMD_PREFIX} ostree --repo=repo fsck
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 
 rm main-files -rf
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main main-files
 cd main-files
 echo "an added file for static deltas" > added-file
 echo "modified file for static deltas" > baz/cow
 rm baz/saucer
-${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s 'static delta test'
+${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s 'static delta test'
 cd ..
 rm main-files -rf
 # Generate delta that we'll use
@@ -317,12 +348,24 @@ repo_init --no-gpg-verify
 ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
 ${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${delta_target} >dry-run-pull.txt
 # Compression can vary, so we support 400-699
-assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts, 0 bytes/[456][0-9][0-9] bytes, 455 bytes total uncompressed'
+delta_dry_run_regexp='Delta update: 0/1 parts, 0 bytes/[456][0-9][0-9] bytes, 455 bytes total uncompressed'
+assert_file_has_content dry-run-pull.txt "${delta_dry_run_regexp}"
 rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
 assert_streq "${prev_rev}" "${rev}"
 ${CMD_PREFIX} ostree --repo=repo fsck
 done
 
+# Test pull via file:/// - this should still use the deltas path for testing
+cd ${test_tmpdir}
+repo_init --no-gpg-verify
+${CMD_PREFIX} ostree --repo=repo remote delete origin
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo
+${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${delta_target} >dry-run-pull.txt
+# See above
+assert_file_has_content dry-run-pull.txt "${delta_dry_run_regexp}"
+echo "ok pull file:// + deltas required"
+
 # Explicitly test delta fetches via ref name as well as commit hash
 for delta_target in main ${new_rev}; do
 cd ${test_tmpdir}
@@ -345,7 +388,7 @@ ${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin main
 ${CMD_PREFIX} ostree --repo=repo fsck
 
 rm checkout-origin-main -rf
-$OSTREE checkout origin:main checkout-origin-main
+$OSTREE checkout ${CHECKOUT_H_ARGS} origin:main checkout-origin-main
 cd checkout-origin-main
 assert_file_has_content firstfile '^first$'
 assert_file_has_content baz/cow "modified file for static deltas"
@@ -397,10 +440,10 @@ echo "ok delta required for revision"
 
 cd ${test_tmpdir}
 rm main-files -rf
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main main-files
 cd main-files
 echo "more added files for static deltas" > added-file2
-${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s 'inline static delta test'
+${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s 'inline static delta test'
 cd ..
 rm main-files -rf
 # Generate new delta that we'll use
@@ -412,7 +455,7 @@ ${CMD_PREFIX} ostree --repo=repo pull origin main
 ${CMD_PREFIX} ostree --repo=repo fsck
 
 rm checkout-origin-main -rf
-$OSTREE checkout origin:main checkout-origin-main
+$OSTREE checkout ${CHECKOUT_H_ARGS} origin:main checkout-origin-main
 cd checkout-origin-main
 assert_file_has_content added-file2 "more added files for static deltas"
 
@@ -420,12 +463,12 @@ echo "ok inline static delta"
 
 cd ${test_tmpdir}
 rm main-files -rf
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout ${CHECKOUT_U_ARG} main main-files
 cd main-files
 # Make a file larger than 16M for testing
 dd if=/dev/zero of=test-bigfile count=1 seek=42678
 echo "further modified file for static deltas" > baz/cow
-${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s '2nd static delta test'
+${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main -s '2nd static delta test'
 cd ..
 rm main-files -rf
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
@@ -436,7 +479,7 @@ ${CMD_PREFIX} ostree --repo=repo pull origin main
 ${CMD_PREFIX} ostree --repo=repo fsck
 
 rm checkout-origin-main -rf
-$OSTREE checkout origin:main checkout-origin-main
+$OSTREE checkout ${CHECKOUT_H_ARGS} origin:main checkout-origin-main
 cd checkout-origin-main
 assert_has_file test-bigfile
 stat --format=%s test-bigfile > bigfile-size
@@ -488,7 +531,7 @@ echo "ok pull repo 404 on dirtree object"
 
 cd ${test_tmpdir}
 repo_init --set=gpg-verify=true
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit \
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} \
   --gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1} -b main \
   -s "A signed commit" --tree=ref=main
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u

tests/pull-test2.sh

@@ -0,0 +1,61 @@
+# This file is to be sourced, not executed
+
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+function repo_init() {
+    cd ${test_tmpdir}
+    rm repo -rf
+    mkdir repo
+    ostree_repo_init repo --mode=${repo_mode}
+    ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@"
+}
+
+repo_init --no-gpg-verify
+
+# See also the copy of this in basic-test.sh
+COMMIT_ARGS=""
+CHECKOUT_U_ARG=""
+CHECKOUT_H_ARGS="-H"
+if is_bare_user_only_repo repo; then
+    COMMIT_ARGS="--canonical-permissions"
+    # Also, since we can't check out uid=0 files we need to check out in user mode
+    CHECKOUT_U_ARG="-U"
+    CHECKOUT_H_ARGS="-U -H"
+else
+    if grep -E -q '^mode=bare-user' repo/config; then
+        CHECKOUT_H_ARGS="-U -H"
+    fi
+fi
+
+echo "1..1"
+cd ${test_tmpdir}
+repo_init --no-gpg-verify
+prev_rev=$(ostree --repo=ostree-srv/repo rev-parse ${remote_ref}^)
+rev=$(ostree --repo=ostree-srv/repo rev-parse ${remote_ref})
+${CMD_PREFIX} ostree --repo=ostree-srv/repo static-delta generate ${remote_ref}
+${CMD_PREFIX} ostree --repo=ostree-srv/repo summary -u
+${CMD_PREFIX} ostree --repo=repo pull origin ${remote_ref}@${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${remote_ref} >dry-run-pull.txt
+assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts, 0 bytes/[45][0-9].[0-9] kB, 1.[678] MB total uncompressed'
+${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${remote_ref}
+final_rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:${remote_ref})
+assert_streq "${rev}" "${final_rev}"
+${CMD_PREFIX} ostree --repo=repo fsck
+echo "ok delta"

tests/test-admin-deploy-uboot.sh

@@ -30,6 +30,10 @@ extra_admin_tests=1
 . $(dirname $0)/admin-test.sh
 
 cd ${test_tmpdir}
+# Note this test actually requires a checksum change to /boot,
+# because adding the uEnv.txt isn't currently covered by the
+# "bootcsum".
+os_repository_new_commit "uboot test" "test upgrade multiple kernel args"
 mkdir -p osdata/usr/lib/ostree-boot
 cat << 'EOF' > osdata/usr/lib/ostree-boot/uEnv.txt
 loaduimage=load mmc ${bootpart} ${loadaddr} ${kernel_image}

tests/test-basic-user.sh

@@ -25,7 +25,7 @@ skip_without_user_xattrs
 
 setup_test_repository "bare-user"
 
-extra_basic_tests=4
+extra_basic_tests=5
 . $(dirname $0)/basic-test.sh
 
 # Reset things so we don't inherit a lot of state from earlier tests
@@ -73,3 +73,29 @@ rm test2-checkout -rf
 $OSTREE checkout -U -H test2-unreadable test2-checkout
 assert_file_has_mode test2-checkout/unreadable 400
 echo "ok bare-user handled unreadable file"
+
+cd ${test_tmpdir}
+mkdir -p components/{dbus,systemd}/usr/{bin,lib}
+echo dbus binary > components/dbus/usr/bin/dbus-daemon
+chmod a+x components/dbus/usr/bin/dbus-daemon
+echo dbus lib > components/dbus/usr/lib/libdbus.so.1
+echo dbus helper > components/dbus/usr/lib/dbus-daemon-helper
+chmod a+x components/dbus/usr/lib/dbus-daemon-helper
+echo systemd binary > components/systemd/usr/bin/systemd
+chmod a+x components/systemd/usr/bin/systemd
+echo systemd lib > components/systemd/usr/lib/libsystemd.so.1
+
+# Make the gid on dbus 81 like fedora
+$OSTREE commit -b component-dbus --owner-uid 0 --owner-gid 81 --tree=dir=components/dbus
+$OSTREE commit -b component-systemd --owner-uid 0 --owner-gid 0 --tree=dir=components/systemd
+rm rootfs -rf
+for component in dbus systemd; do
+    $OSTREE checkout -U -H component-${component} --union rootfs
+done
+echo 'some rootfs data' > rootfs/usr/lib/cache.txt
+$OSTREE commit -b rootfs --link-checkout-speedup --tree=dir=rootfs
+$OSTREE ls rootfs /usr/bin/systemd >ls.txt
+assert_file_has_content ls.txt '^-007.. 0 0 .*/usr/bin/systemd'
+$OSTREE ls rootfs /usr/lib/dbus-daemon-helper >ls.txt
+assert_file_has_content ls.txt '^-007.. 0 81 .*/usr/lib/dbus-daemon-helper'
+echo "ok bare-user link-checkout-speedup maintains uids"

tests/test-help.sh

@@ -46,7 +46,8 @@ test_recursive() {
     test_usage_output out "$cmd"
     assert_file_empty err
 
-    builtins=`sed -n '/^Builtin \("[^"]*" \)\?Commands:$/,/^$/p' <out | tail -n +2`
+    # Select the list of commands, for each line, remove the leading spaces, and take the first word(command) of each line
+    builtins=`sed -n '/^Builtin \("[^"]*" \)\?Commands:$/,/^$/p' <out | tail -n +2 | sed -e 's/^[[:space:]]*//' | cut -d " " -f1`
     if [ "$builtins" != "" ] ; then
 
         found_subcommands=1

tests/test-pull-bareuseronly.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+skip_without_user_xattrs
+setup_fake_remote_repo1 "archive" "--canonical-permissions"
+
+repo_mode=bare-user-only
+. ${test_srcdir}/pull-test.sh

tests/test-pull-corruption.sh

@@ -89,7 +89,7 @@ if ! skip_one_without_user_xattrs; then
     if ${CMD_PREFIX} ostree --repo=repo pull-local --untrusted ostree-srv/gnomerepo main 2>err.txt; then
         assert_not_reached "pull-local --untrusted worked?"
     fi
-    assert_file_has_content err.txt "Corrupted commit object ${corruptrev}.*actual checksum is ${rev}"
+    assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected='${corruptrev}' actual='${rev}'"
 
     rm repo err.txt -rf
     ostree_repo_init repo --mode=bare-user
@@ -97,6 +97,6 @@ if ! skip_one_without_user_xattrs; then
     if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.txt; then
         assert_not_reached "pull unexpectedly succeeded!"
     fi
-    assert_file_has_content err.txt "Corrupted commit object ${corruptrev}.*actual checksum is ${rev}"
+    assert_file_has_content_literal err.txt "Corrupted commit object; checksum expected='${corruptrev}' actual='${rev}'"
     echo "ok pull commit corruption"
 fi

tests/test-pull2-bareuseronly.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 Colin Walters <walters@verbum.org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo2 "archive" "--canonical-permissions"
+
+repo_mode=bare-user-only
+. ${test_srcdir}/pull-test2.sh

tests/test-refs.sh

@@ -23,7 +23,7 @@ set -euo pipefail
 
 setup_fake_remote_repo1 "archive"
 
-echo '1..2'
+echo '1..5'
 
 cd ${test_tmpdir}
 mkdir repo
@@ -88,6 +88,35 @@ if ${CMD_PREFIX} ostree --repo=repo refs foo/ctest --create=ctest; then
     assert_not_reached "refs --create unexpectedly succeeded in overwriting an existing prefix!"
 fi
 
+# https://github.com/ostreedev/ostree/issues/1285
+# One tool was creating .latest_rsync files in each dir, let's ignore stuff like
+# that.
+echo '👻' > repo/refs/heads/.spooky_and_hidden
+${CMD_PREFIX} ostree --repo=repo refs > refs.txt
+assert_not_file_has_content refs.txt 'spooky'
+${CMD_PREFIX} ostree --repo=repo refs ctest --create=exampleos/x86_64/standard
+echo '👻' > repo/refs/heads/exampleos/x86_64/.further_spooky_and_hidden
+${CMD_PREFIX} ostree --repo=repo refs > refs.txt
+assert_file_has_content refs.txt 'exampleos/x86_64/standard'
+assert_not_file_has_content refs.txt 'spooky'
+rm repo/refs/heads/exampleos -rf
+echo "ok hidden refs"
+
+for ref in '.' '-' '.foo' '-bar' '!' '!foo'; do
+    if ${CMD_PREFIX} ostree --repo=repo refs ctest --create=${ref} 2>err.txt; then
+        fatal "Created ref ${ref}"
+    fi
+    assert_file_has_content err.txt 'Invalid ref'
+done
+echo "ok invalid refs"
+
+for ref in 'org.foo.bar/x86_64/standard-blah'; do
+    ostree --repo=repo refs ctest --create=${ref}
+    ostree --repo=repo rev-parse ${ref} >/dev/null
+    ostree --repo=repo refs --delete ${ref}
+done
+echo "ok valid refs with chars '._-'"
+
 #Check to see if any uncleaned tmp files were created after failed --create
 ${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.create1
 assert_file_has_content refscount.create1 "^5$"

tests/test-repo-finder-config.c

@@ -66,8 +66,6 @@ static void
 teardown (Fixture       *fixture,
           gconstpointer  test_data)
 {
-  g_autoptr(GError) error = NULL;
-
   /* Recursively remove the temporary directory. */
   (void)glnx_tmpdir_delete (&fixture->tmpdir, NULL, NULL);
 

tests/test-repo-finder-mount.c

@@ -154,7 +154,7 @@ assert_create_repos_dir (Fixture      *fixture,
                          int          *out_repos_dfd,
                          GMount      **out_mount)
 {
-  glnx_fd_close int repos_dfd = -1;
+  glnx_autofd int repos_dfd = -1;
   g_autoptr(GError) error = NULL;
 
   g_autofree gchar *path = g_build_filename (mount_root_name, ".ostree", "repos.d", NULL);
@@ -242,7 +242,7 @@ assert_create_repo_dir (Fixture     *fixture,
                         gchar      **out_uri,
                         ...)
 {
-  glnx_fd_close int ref_dfd = -1;
+  glnx_autofd int ref_dfd = -1;
   g_autoptr(OstreeRepo) repo = NULL;
   g_autoptr(GError) error = NULL;
   va_list args;
@@ -318,9 +318,9 @@ test_repo_finder_mount_mixed_mounts (Fixture       *fixture,
   g_autoptr(GMount) repo1_mount = NULL;
   g_autoptr(GMount) repo2_mount = NULL;
   g_autoptr(GFile) non_removable_root = NULL;
-  glnx_fd_close int no_repos_repos = -1;
-  glnx_fd_close int repo1_repos = -1;
-  glnx_fd_close int repo2_repos = -1;
+  glnx_autofd int no_repos_repos = -1;
+  glnx_autofd int repo1_repos = -1;
+  glnx_autofd int repo2_repos = -1;
   g_autoptr(OstreeRepo) repo1_repo_a = NULL, repo1_repo_b = NULL;
   g_autoptr(OstreeRepo) repo2_repo_a = NULL;
   g_autofree gchar *repo1_repo_a_uri = NULL, *repo1_repo_b_uri = NULL;
@@ -464,7 +464,7 @@ test_repo_finder_mount_well_known (Fixture       *fixture,
   g_autoptr(GError) error = NULL;
   g_autoptr(GList) mounts = NULL;  /* (element-type OstreeMockMount)  */
   g_autoptr(GMount) mount = NULL;
-  glnx_fd_close int repos = -1;
+  glnx_autofd int repos = -1;
   g_autoptr(OstreeRepo) repo_a = NULL, repo_b = NULL;
   g_autofree gchar *repo_a_uri = NULL, *repo_b_uri = NULL;
   g_autofree gchar *ref_a_checksum = NULL, *ref_b_checksum = NULL;

tests/test-symbols.sh

@@ -52,7 +52,7 @@ echo 'ok documented symbols'
 
 # ONLY update this checksum in release commits!
 cat > released-sha256.txt <<EOF
-809b49ee0c3a7c230213a00d8c021c3a0ebb91988e39225ded83ef500b9c5eba  ${released_syms}
+f8c387ff263f46ac5b8f56c05d37385fb0001a0494998925fd5e992399360386  ${released_syms}
 EOF
 sha256sum -c released-sha256.txt