tests: Rework tests/installed → tests/kola

Previously we made an effort to use the [Fedora Standard Test Interface](https://docs.fedoraproject.org/en-US/ci/standard-test-interface/).
This effort was not very successful; the primary thing that
it really died on is Ansible just didn't support rebooting
very well.  I think that's since gotten better, but even
then, Ansible wasn't the best thing for a test framework
for us anyways.

In the meantime Fedora CoreOS happened emphasizing Ignition
and not "post-hoc reconciliation" models like Ansible over
ssh.

And, [coreos-assembler](https://github.com/coreos/coreos-assembler) happened too.

Furthermore, we really need to test OSTree's interaction
with Ignition as we've invented several special things there.

Then most recently, I've been working on having
cosa/kola support running externally defined tests:
https://github.com/coreos/coreos-assembler/pull/1215

There's a lot of things to clean up after this but at least this
works for me:

```
$ cd /srv/fcos
$ cosa kola run -- --parallel 4 --output-dir tmp/kola -E ~/src/github/ostreedev/ostree/ 'ext.ostree.*'
```

NOTE: This *does not* drop ostree binaries into the target.  See:
https://github.com/coreos/coreos-assembler/pull/1252#issuecomment-600623315

This drops our dependency on Python in the installed tests, and
also fixes a few bugs that came up.

I disabled the `itest-bare-user-root.sh` one because it's
entangled with the shell script infrastructure for the unit tests.

tests/kola/nondestructive/data

@@ -0,0 +1 @@
+..

tests/kola/nondestructive/itest-bare-unit.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Run test-basic.sh as root.
+# https://github.com/ostreedev/ostree/pull/1199
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+
+fatal "FIXME - need to also sync over the installed tests"
+
+date
+# These tests sort of bypass the installed-tests spec;
+# fixing that would require installing g-d-t-r, though
+# more ideally we architect things with a "control" container
+# distinct from the host.
+export G_TEST_SRCDIR=$(realpath $dn/../../..)
+
+# Use /var/tmp to hopefully use XFS + O_TMPFILE etc.
+prepare_tmpdir /var/tmp
+trap _tmpdir_cleanup EXIT
+/usr/libexec/installed-tests/libostree/test-basic.sh
+/usr/libexec/installed-tests/libostree/test-basic-c
+date
+
+# Test error message when opening a non-world-readable object
+# https://github.com/ostreedev/ostree/issues/1562
+rm repo files -rf
+chmod a+rx .
+ostree --repo=repo init --mode=bare
+mkdir files
+touch files/unreadable
+chmod 0 files/unreadable
+ostree --repo=repo commit -b testbranch --tree=dir=files
+# We should be able to read as root due to CAP_DAC_OVERRIDE
+ostree --repo=repo cat testbranch /unreadable >/dev/null
+if setpriv --reuid bin --regid bin --clear-groups ostree --repo=repo cat testbranch /unreadable 2>err.txt; then
+    fatal "Listed unreadable object as non-root"
+fi
+assert_file_has_content err.txt "Opening content object.*openat: Permission denied"
+
+date

tests/kola/nondestructive/itest-bare-user-root.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Tests of the "raw ostree" functionality using the host's ostree repo as uid 0.
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+
+echo "1..1"
+date
+
+prepare_tmpdir
+ostree --repo=repo init --mode=bare-user
+mkdir -p components/{dbus,systemd}/usr/{bin,lib}
+echo dbus binary > components/dbus/usr/bin/dbus-daemon
+chmod a+x components/dbus/usr/bin/dbus-daemon
+echo dbus lib > components/dbus/usr/lib/libdbus.so.1
+echo dbus helper > components/dbus/usr/lib/dbus-daemon-helper
+chmod a+x components/dbus/usr/lib/dbus-daemon-helper
+echo systemd binary > components/systemd/usr/bin/systemd
+chmod a+x components/systemd/usr/bin/systemd
+echo systemd lib > components/systemd/usr/lib/libsystemd.so.1
+
+# Make the gid on dbus 81 like fedora, also ensure no xattrs
+ostree --repo=repo commit --no-xattrs -b component-dbus --owner-uid 0 --owner-gid 81 --tree=dir=components/dbus
+ostree --repo=repo commit --no-xattrs -b component-systemd --owner-uid 0 --owner-gid 0 --tree=dir=components/systemd
+rm rootfs -rf
+for component in dbus systemd; do
+    ostree --repo=repo checkout -U -H component-${component} --union rootfs
+done
+echo 'some rootfs data' > rootfs/usr/lib/cache.txt
+# Commit using the host's selinux policy
+ostree --repo=repo commit --selinux-policy / -b rootfs --link-checkout-speedup --tree=dir=rootfs
+ostree --repo=repo ls rootfs /usr/bin/systemd >ls.txt
+assert_file_has_content ls.txt '^-007.. 0 0 .*/usr/bin/systemd'
+ostree --repo=repo ls -X rootfs /usr/lib/dbus-daemon-helper >ls.txt
+assert_file_has_content ls.txt '^-007.. 0 81 .*security\.selinux.*/usr/lib/dbus-daemon-helper'
+assert_not_file_has_content ls.txt 'user\.ostreemeta'
+echo "ok bare-user link-checkout-speedup with modified xattrs maintains uids"
+date

tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Test that initializing a bare-user repo on tmpfs fails
+# Maybe at some point this will be fixed in the kernel
+# but I doubt it'll be soon
+# https://www.spinics.net/lists/linux-mm/msg109775.html
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+
+prepare_tmpdir
+trap _tmpdir_cleanup EXIT
+date
+
+mkdir mnt
+mount -t tmpfs tmpfs mnt
+if ostree --repo=mnt/repo init --mode=bare-user 2>err.txt; then
+    umount mnt
+    assert_not_reached "bare-user on tmpfs worked?"
+fi
+umount mnt
+assert_file_has_content err.txt "Operation not supported"
+date

tests/kola/nondestructive/itest-payload-link.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# SPDX-License-Identifier: LGPL-2.0+
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+
+echo "1..1"
+date
+
+# Use /var/tmp so we have O_TMPFILE etc.
+prepare_tmpdir /var/tmp
+trap _tmpdir_cleanup EXIT
+# We use this user down below, it needs access too
+setfacl -d -m u:bin:rwX .
+setfacl -m u:bin:rwX .
+ostree --repo=repo init --mode=archive
+echo -e '[archive]\nzlib-level=1\n' >> repo/config
+
+mkdir content
+cd content
+dd if=/dev/urandom of=bigobject bs=4k count=2560
+cp --reflink=auto bigobject bigobject2
+# Different metadata, same content
+chown bin:bin bigobject2
+cd ..
+ostree --repo=repo commit -b dupobjects --consume --selinux-policy=/ --tree=dir=content
+ostree --repo=repo summary -u
+
+run_tmp_webserver $(pwd)/repo
+
+origin=$(cat ${test_tmpdir}/httpd-address)
+
+cleanup() {
+    cd ${test_tmpdir}
+    for mnt in ${mnts:-}; do
+        umount ${mnt} || true
+    done
+    for blkdev in ${blkdevs:-}; do
+        losetup -d ${blkdev} || true
+    done
+}
+trap cleanup EXIT
+
+truncate -s 2G testblk1.img
+blkdev1=$(losetup --find --show $(pwd)/testblk1.img)
+blkdevs="${blkdev1}"
+# This filesystem must support reflinks
+mkfs.xfs -m reflink=1 ${blkdev1}
+mkdir mnt1
+mount ${blkdev1} mnt1
+mnts=mnt1
+
+truncate -s 2G testblk2.img
+blkdev2=$(losetup --find --show $(pwd)/testblk2.img)
+blkdevs="${blkdev1} ${blkdev2}"
+mkfs.xfs -m reflink=1 ${blkdev2}
+mkdir mnt2
+mount ${blkdev2} mnt2
+mnts="mnt1 mnt2"
+
+cd mnt1
+# See above for setfacl rationale
+setfacl -d -m u:bin:rwX .
+setfacl -m u:bin:rwX .
+ls -al .
+runuser -u bin mkdir foo
+runuser -u bin touch foo/bar
+ls -al foo
+
+# Test that reflink is really there (not just --reflink=auto)
+touch a
+cp --reflink a b
+mkdir repo
+ostree --repo=repo init
+ostree config --repo=repo set core.payload-link-threshold 0
+ostree --repo=repo remote add origin --set=gpg-verify=false ${origin}
+ostree --repo=repo pull --disable-static-deltas origin dupobjects
+find repo -type l -name '*.payload-link' >payload-links.txt
+assert_streq "$(wc -l < payload-links.txt)" "1"
+
+cat payload-links.txt | while read i; do
+    payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link)
+    payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1)
+    assert_streq "${payload_checksum}" "${payload_checksum_calculated}"
+done
+echo "ok payload link"
+
+ostree --repo=repo checkout dupobjects content
+# And another object which differs just in metadata
+cp --reflink=auto content/bigobject{,3}
+chown operator:0 content/bigobject3
+cat >unpriv-child-repo.sh <<EOF
+#!/bin/bash
+# Check that commit to an user repo that has a parent still works
+set -xeuo pipefail
+ostree --repo=child-repo init --mode=bare-user
+ostree --repo=child-repo remote add origin --set=gpg-verify=false ${origin}
+ostree --repo=child-repo config set core.parent $(pwd)/repo
+ostree --repo=child-repo commit -b test content
+EOF
+chmod a+x unpriv-child-repo.sh
+runuser -u bin ./unpriv-child-repo.sh
+find child-repo -type l -name '*.payload-link' >payload-links.txt
+assert_streq "$(wc -l < payload-links.txt)" "0"
+rm content -rf
+
+echo "ok reflink unprivileged with parent repo"
+
+# We can't reflink across devices though
+cd ../mnt2
+ostree --repo=repo init --mode=archive
+ostree --repo=repo config set core.parent $(cd ../mnt1/repo && pwd)
+ostree --repo=../mnt1/repo checkout dupobjects content
+ostree --repo=repo commit -b dupobjects2 --consume --tree=dir=content
+ostree --repo=repo pull --disable-static-deltas origin dupobjects
+find repo -type l -name '*.payload-link' >payload-links.txt
+assert_streq "$(wc -l < payload-links.txt)" "0"
+
+echo "ok payload link across devices"
+
+date

tests/kola/nondestructive/itest-pull-space.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+# Test min-free-space-percent using loopback devices
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+date
+
+prepare_tmpdir
+truncate -s 20MB testblk.img
+blkdev=$(losetup --find --show $(pwd)/testblk.img)
+mkfs.xfs ${blkdev}
+mkdir mnt
+mount ${blkdev} mnt
+
+# first test min-free-space-percent
+ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
+if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then
+    fatal "succeeded in doing a pull with no free space"
+fi
+assert_file_has_content err.txt "min-free-space-percent"
+echo "ok min-free-space-percent"
+
+# now test min-free-space-size
+rm -rf mnt/repo
+ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
+echo 'min-free-space-size=10MB' >> mnt/repo/config
+if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then
+    fatal "succeeded in doing a pull with no free space"
+fi
+assert_file_has_content err.txt "min-free-space-size"
+echo "ok min-free-space-size (error)"
+
+
+# min-free-space-size success
+ostree --repo=repo init --mode=bare-user
+echo 'fsync=false' >> repo/config
+echo 'min-free-space-size=1MB' >> repo/config
+ostree --repo=repo pull-local /ostree/repo ${host_commit}
+echo "ok min-free-space-size (success)"
+
+# metadata object write should succeed even if free space is lower than
+# min-free-space value
+rm -rf mnt/repo
+ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
+echo 'min-free-space-size=10MB' >> mnt/repo/config
+ostree --repo=mnt/repo pull-local --commit-metadata-only /ostree/repo ${host_commit}
+echo "ok metadata write even when free space is lower than min-free-space value"
+
+rm -rf mnt/repo
+
+# Test min-free-space-size on deltas
+
+#helper function copied from test-delta.sh
+get_assert_one_direntry_matching() {
+    local path=$1
+    local r=$2
+    local child=""
+    local bn
+    for p in ${path}/*; do
+  bn=$(basename $p)
+  if ! echo ${bn} | grep -q "$r"; then
+      continue
+  fi
+  if test -z "${child}"; then
+      child=${bn}
+  else
+      assert_not_reached "Expected only one child matching ${r} in ${path}";
+  fi
+    done
+    if test -z "${child}"; then
+  assert_not_reached "Failed to find child matching ${r}"
+    fi
+    echo ${child}
+}
+
+mkdir mnt/repo1 mnt/repo2
+ostree --repo=mnt/repo1 init --mode=bare-user
+ostree --repo=mnt/repo2 init --mode=bare-user
+mkdir files
+echo "hello" >> files/test.txt
+truncate -s 2MB files/test.txt
+
+host_commit=$(ostree --repo=mnt/repo1 commit -b test -s test --tree=dir=files)
+
+origrev=$(ostree --repo=mnt/repo1 rev-parse test)
+ostree --repo=mnt/repo1 static-delta generate --empty --to=${origrev}
+echo 'fsync=false' >> mnt/repo2/config
+echo 'min-free-space-size=12MB' >> mnt/repo2/config
+
+deltaprefix=$(get_assert_one_direntry_matching mnt/repo1/deltas '.')
+deltadir=$(get_assert_one_direntry_matching mnt/repo1/deltas/${deltaprefix} '.')
+
+# Try to pull delta and trigger error
+if ostree --repo=mnt/repo2 static-delta apply-offline mnt/repo1/deltas/${deltaprefix}/${deltadir} 2>err.txt; then
+    fatal "succeeded in doing a delta pull with no free space"
+fi
+assert_file_has_content err.txt "min-free-space-size"
+echo "OK min-free-space-size delta pull (error)"
+
+# Re-adjust min-free-space-size so that delta pull succeeds
+sed -i s/min-free-space-size=12MB/min-free-space-size=1MB/g mnt/repo2/config
+rm -rf mnt/repo2/deltas
+ostree --repo=mnt/repo2 static-delta apply-offline mnt/repo1/deltas/${deltaprefix}/${deltadir}
+
+echo "OK min-free-space-size delta pull (success)"
+rm -rf files
+
+umount mnt
+losetup -d ${blkdev}
+rm testblk.img
+
+date

tests/kola/nondestructive/itest-pull.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Using the host ostree, test HTTP pulls
+
+set -xeuo pipefail
+
+# FIXME: https://github.com/ostreedev/ostree/pull/1548
+exit 0
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+date
+
+prepare_tmpdir /var/tmp
+trap _tmpdir_cleanup EXIT
+
+# Take the host's ostree, and make it archive
+mkdir repo
+ostree --repo=repo init --mode=archive
+echo -e '[archive]\nzlib-level=1\n' >> repo/config
+host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,')
+log_timestamps() {
+    date
+    "$@"
+    date
+}
+log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit}
+ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref}
+
+run_tmp_webserver $(pwd)/repo
+# Now test pulling via HTTP (no deltas) to a new bare-user repo
+ostree --repo=bare-repo init --mode=bare-user
+ostree --repo=bare-repo remote add origin --set=gpg-verify=false $(cat ${test_tmpdir}/httpd-address)
+log_timestamps ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref}
+
+echo "ok pull"
+
+# fsck marks commits partial
+# https://github.com/ostreedev/ostree/pull/1533
+for d in $(find bare-repo/objects/ -maxdepth 1 -type d); do
+    (find ${d} -name '*.file' || true) | head -20 | xargs rm -f
+done
+date
+if ostree --repo=bare-repo fsck |& tee fsck.txt; then
+    fatal "fsck unexpectedly succeeded"
+fi
+date
+assert_streq $(grep -cE -e 'Marking commit as partial' fsck.txt) "1"
+log_timestamps ostree --repo=bare-repo pull origin ${host_nonremoteref}
+# Don't need a full fsck here
+ostree --repo=bare-repo ls origin:${host_nonremoteref} >/dev/null
+
+rm bare-repo repo -rf
+
+# Try copying the host's repo across a mountpoint for direct
+# imports.
+cd ${test_tmpdir}
+mkdir tmpfs mnt
+mount --bind tmpfs mnt
+cd mnt
+ostree --repo=repo init --mode=bare
+log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit}
+log_timestamps ostree --repo=repo fsck
+cd ..
+
+# Also, we shouldn't copy xattrs on metadata objects
+commit_path=objects/${host_commit:0:2}/${host_commit:2}.commit
+ostree --repo=testarchive init --mode=archive
+ostree --repo=testarchive pull-local --commit-metadata-only /ostree/repo ${host_commit}
+setfattr -n user.ostreetesting -v hello testarchive/${commit_path}
+ostree --repo=mnt/testarchive2 init --mode=archive
+ostree --repo=mnt/testarchive2 pull-local --commit-metadata-only testarchive ${host_commit}
+if getfattr -m user.ostreetesting mnt/testarchive2/${commit_path} 2>/dev/null; then
+    fatal "copied metadata xattr"
+fi
+echo "ok no metadata xattr copy"
+
+umount mnt
+
+# Cleanup
+kill -TERM $(cat ${test_tmpdir}/httpd-pid)
+echo "ok"
+date

tests/kola/nondestructive/itest-remotes.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Test that we didn't regress /etc/ostree/remotes.d handling
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libinsttest.sh
+date
+
+prepare_tmpdir
+trap _tmpdir_cleanup EXIT
+
+ostree remote list > remotes.txt
+if ! test -s remotes.txt; then
+    assert_not_reached "no ostree remotes"
+fi
+date

tests/kola/nondestructive/libtest-core.sh

@@ -0,0 +1 @@
+../libtest-core.sh