tests: Merge installed/ and fedora-str/ directories

Let's be opinionated now, and our installed/ test story *is*
Ansible/STR.  Merge `tests/fedora-str` into `tests/installed/`.

Rework the nondestructive tests into a separate playbook run, and parallelize
them for more efficiency.

The destructive tests are also changed to use Ansible more.

Add a higher level `run.sh` entrypoint and update the `README.md`
with some useful tips.

Closes: #1513
Approved by: jlebon

.papr.yml

@@ -7,24 +7,29 @@ branches:
 context: FAH27-insttests
 required: true
 
-container:
+# FIXME; temporary workaround
-  image: registry.fedoraproject.org/fedora:27
+# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162
+host:
+  distro: fedora/27/atomic
+  specs:
+    ram: 4096
+#container:
+#  image: registry.fedoraproject.org/fedora:27
 
 tests:
-  - cd tests/fedora-str && ../../ci/build-rpm.sh
+  - cd /etc/yum.repos.d/ && curl -L -O https://copr.fedorainfracloud.org/coprs/walters/oci-kvm-hook/repo/fedora-27/walters-oci-kvm-hook-fedora-27.repo
-  - ./tests/fedora-str/provision.sh
+  - rpm-ostree install oci-kvm-hook && rpm-ostree ex livefs
-  # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
+  - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/fah27-insttests.sh"
-  - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
-  - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/sysinstall-tests.yml
 
 artifacts:
-  - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log
+  - tests/installed/artifacts/
-  - tests/fedora-str/artifacts/installed-tests.log
 
 ---
 
 # This suite skips the RPMs and does the build+unit tests in a container
-inherit: true
+inherit: false
+container:
+  image: registry.fedoraproject.org/fedora:27
 context: f27-primary
 env:
   # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain

ci/fah27-insttests.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+set -xeuo pipefail
+
+./tests/installed/provision.sh
+# TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
+cd tests/installed
+curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
+exec env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./run.sh

tests/fedora-str/README.md

@@ -1,2 +0,0 @@
-This directory holds tests that use the
-[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface).

tests/fedora-str/provision.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/bash
-set -xeuo pipefail
-
-dn=$(dirname $0)
-. ${dn}/../../ci/libpaprci/libbuild.sh
-
-pkg_upgrade
-pkg_install git rsync openssh-clients ansible standard-test-roles

tests/fedora-str/sysinstall-tests.yml

@@ -1,31 +0,0 @@
-# This entrypoint right now just runs the sysinstalled-tests.
----
-- hosts: localhost
-  tags:
-  - atomic
-  remote_user: root
-  vars:
-    use_git_build: True
-    tests: ""
-  tasks:
-    - import_tasks: overlay-git.yml
-      when: use_git_build
-    # Next copy all of the tests/ directory
-    - name: Copy test data
-      synchronize: src=../../ dest=/root/tests/ archive=yes
-    # Down the line perhaps do each log file separately?
-    - name: Run shell script sysinstalled tests
-      shell: /root/tests/installed/run.sh &> /root/installed-tests.log
-      environment:
-        TESTS: "{{ tests }}"
-      register: sysinstalled_result
-      failed_when: False
-    - name: Fetch sysinstalled results
-      fetch:
-        src: /root/installed-tests.log
-        dest: artifacts/installed-tests.log
-        flat: yes
-    - name: Assert that sysinstalled tests succeeded
-      when: sysinstalled_result.rc != 0
-      fail:
-        msg: "sysinstalled tests failed"

tests/installed/README.md

@@ -1,2 +1,15 @@
-This suite of tests is currently run from redhat-ci;
+This directory holds tests that use the
-they're intended to run as root.
+[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface).
+
+The high level structure is that we take a qcow2 file, inject
+built RPMs into it, and then use Ansible to run tests.
+
+See `.papr.yml` for canonical usage.
+
+For local development, you should cache the qcow2 somewhere
+stable (outside of this git repo).  Also note that `../ci/build-rpms.sh`
+does *not* pick up uncommitted changes!  Stated more strongly, you
+currently need to run `build-rpms.sh` after every change.
+
+To run just a specific test, use e.g.:
+`env TEST_SUBJECTS=/path/to/qcow2 ./playbook-run.sh -e tests=.*pull nondestructive.yml`

tests/installed/destructive.yml

@@ -0,0 +1,35 @@
+# This entrypoint right now just runs the sysinstalled-tests.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/destructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run destructive tests
+          shell: "{{ item }} &> /root/logs/$(basename {{ item }}).log"
+          with_items:
+            - "{{ selected_tests }}"
+      always:
+        - synchronize:
+            src: /root/logs/
+            dest: artifacts/installed-destructive
+            mode: pull

tests/installed/destructive/README.md

@@ -0,0 +1,5 @@
+This suite of tests is run from PAPR. Everything in here is destructive; it's
+recommended to only run them in disposable virtual machines.  This is done
+in `tests/fedora-str/sysinstalled-tests.yml`, which currently uses a single VM
+and runs the tests serially.  It's likely in the future this will be changed
+to do one VM per test.

tests/installed/destructive/itest-bare-root.sh

@@ -5,9 +5,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..2"
+date
 
 cd /ostree/repo/tmp
 rm co -rf
@@ -40,3 +41,4 @@ rm co -rf
 rm co-testref -rf
 
 echo "ok xattrs"
+date

tests/installed/destructive/itest-deploy-selinux.sh

@@ -5,8 +5,9 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 # Create a new deployment
 ostree admin deploy --karg-proc-cmdline ${host_refspec}
 new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1
@@ -54,3 +55,4 @@ assert_file_has_content_literal bootlsz.txt 'system_u:object_r:boot_t:s0 initram
 
 ostree admin undeploy 0
 ostree refs --delete test-label
+date

tests/installed/destructive/itest-label-selinux.sh

@@ -5,8 +5,9 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 cd /ostree/repo/tmp
 rm co -rf
 ostree checkout -H ${host_refspec} co
@@ -85,3 +86,4 @@ assert_streq "${oldcon}" "${newcon}"
 rm co -rf
 ostree refs --delete testbranch
 echo "ok checkout selinux and skip-list"
+date

tests/installed/execute_batch.yml

@@ -0,0 +1,22 @@
+#####################
+# execute_batch.yml
+#####################
+- name: Begin async command execution
+  shell: "{{ async_item }} &> {{ logdir }}/{{ async_item|basename }}.log"
+  # 10 minutes; the PAPR tester generally times out before that
+  async: 600
+  poll: 0
+  with_items: "{{ async_commands }}"
+  loop_control:
+    loop_var: "async_item"
+  register: async_results
+
+- name: Check async command status
+  async_status:
+    jid: "{{ async_result_item.ansible_job_id }}"
+  with_items: "{{ async_results.results }}"
+  loop_control:
+    loop_var: "async_result_item"
+  register: async_poll_results
+  until: async_poll_results.finished
+  retries: 240

tests/installed/libinsttest.sh

@@ -20,7 +20,7 @@
 # Boston, MA 02111-1307, USA.
 
 dn=$(dirname $0)
-. ${dn}/libtest-core.sh
+. ${dn}/../libtest-core.sh
 
 # Copy of bits from tap-test
 test_tmpdir=

tests/installed/nondestructive.yml

@@ -0,0 +1,40 @@
+# Nondestructive sysinstalled tests, run in parallel.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+    # Arbitrary...we want some parallelism
+    batching_factor: 4
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/nondestructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run nondestructive tests
+          vars:
+            logdir: /root/logs
+          async_commands: "{{ item }}"
+          include_tasks: execute_batch.yml
+          with_items:
+            - "{{ selected_tests | batch('{{ batching_factor }}') | list }}"
+      always:
+        - synchronize:
+            src: /root/logs
+            dest: artifacts/installed-nondestructive
+            mode: pull

tests/installed/nondestructive/itest-bare-unit.sh

@@ -6,16 +6,18 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 # These tests sort of bypass the installed-tests spec;
 # fixing that would require installing g-d-t-r, though
 # more ideally we architect things with a "control" container
 # distinct from the host.
-export G_TEST_SRCDIR=$(realpath $dn/../..)
+export G_TEST_SRCDIR=$(realpath $dn/../../..)
 
 # Use /var/tmp to hopefully use XFS + O_TMPFILE etc.
 prepare_tmpdir /var/tmp
 trap _tmpdir_cleanup EXIT
 /usr/libexec/installed-tests/libostree/test-basic.sh
 /usr/libexec/installed-tests/libostree/test-basic-c
+date

tests/installed/nondestructive/itest-bare-user-root.sh

@@ -5,9 +5,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
+date
 
 prepare_tmpdir
 ostree --repo=repo init --mode=bare-user
@@ -37,3 +38,4 @@ ostree --repo=repo ls -X rootfs /usr/lib/dbus-daemon-helper >ls.txt
 assert_file_has_content ls.txt '^-007.. 0 81 .*security.selinux.*/usr/lib/dbus-daemon-helper'
 assert_not_file_has_content ls.txt 'user.ostreemeta'
 echo "ok bare-user link-checkout-speedup with modified xattrs maintains uids"
+date

tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh

@@ -8,10 +8,11 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
+date
 
 mkdir mnt
 mount -t tmpfs tmpfs mnt
@@ -21,3 +22,4 @@ if ostree --repo=mnt/repo init --mode=bare-user 2>err.txt; then
 fi
 umount mnt
 assert_file_has_content err.txt "Operation not supported"
+date

tests/installed/nondestructive/itest-payload-link.sh

@@ -22,9 +22,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
+date
 
 # Use /var/tmp so we have O_TMPFILE etc.
 prepare_tmpdir /var/tmp
@@ -86,3 +87,4 @@ cat payload-links.txt | while read i; do
 done
 set -x
 echo "ok pull creates .payload-link"
+date

tests/installed/nondestructive/itest-pull-space.sh

@@ -4,21 +4,24 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
+date
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
 
 cd ${test_tmpdir}
-truncate -s 100MB testblk.img
+truncate -s 20MB testblk.img
 blkdev=$(losetup --find --show $(pwd)/testblk.img)
 mkfs.xfs ${blkdev}
 mkdir mnt
 mount ${blkdev} mnt
 ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
 if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then
     fatal "succeeded in doing a pull with no free space"
 fi
 assert_file_has_content err.txt "min-free-space-percent"
 umount mnt
 losetup -d ${blkdev}
+date

tests/installed/nondestructive/itest-pull.sh

@@ -5,7 +5,8 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
+date
 
 prepare_tmpdir /var/tmp
 trap _tmpdir_cleanup EXIT
@@ -15,14 +16,19 @@ mkdir repo
 ostree --repo=repo init --mode=archive
 echo -e '[archive]\nzlib-level=1\n' >> repo/config
 host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,')
-ostree --repo=repo pull-local /ostree/repo ${host_commit}
+log_timestamps() {
+    date
+    "$@"
+    date
+}
+log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit}
 ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref}
 
 run_tmp_webserver $(pwd)/repo
 # Now test pulling via HTTP (no deltas) to a new bare-user repo
 ostree --repo=bare-repo init --mode=bare-user
 ostree --repo=bare-repo remote add origin --set=gpg-verify=false $(cat ${test_tmpdir}/httpd-address)
-ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref}
+log_timestamps ostree --repo=bare-repo pull --disable-static-deltas origin ${host_nonremoteref}
 
 rm bare-repo repo -rf
 
@@ -33,7 +39,11 @@ mkdir tmpfs mnt
 mount --bind tmpfs mnt
 cd mnt
 ostree --repo=repo init --mode=bare
-ostree --repo=repo pull-local /ostree/repo ${host_commit}
+log_timestamps ostree --repo=repo pull-local /ostree/repo ${host_commit}
-ostree --repo=repo fsck
+log_timestamps ostree --repo=repo fsck
 cd ..
 umount mnt
+
+kill -TERM $(cat ${test_tmpdir}/httpd-pid)
+echo "ok"
+date

tests/installed/nondestructive/itest-remotes.sh

@@ -5,7 +5,8 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
+date
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
@@ -14,3 +15,4 @@ ostree remote list > remotes.txt
 if ! test -s remotes.txt; then
     assert_not_reached "no ostree remotes"
 fi
+date

tests/installed/nondestructive/libtest-core.sh

@@ -0,0 +1 @@
+../libtest-core.sh

tests/installed/overlay-git.yml

@@ -6,7 +6,7 @@
 - set_fact:
     ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}"
 - name: Copy locally built RPMs
-  synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes
+  synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes
 - shell: ostree admin unlock || true
 # Install the RPMs we already have.  For the test suite we use rpm2cpio
 # since it depends on libsoup, but we're not using that yet for the sysinstalled tests

tests/installed/playbook-run.sh

@@ -17,7 +17,9 @@ somewhere persistent.
 EOF
     exit 1
 fi
-ls -al ${TEST_SUBJECTS}
+for subj in ${TEST_SUBJECTS}; do
+    ls -al ${subj} && file ${subj}
+done
 
 # This is required
 rpm -q standard-test-roles

tests/installed/provision.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/bash
+set -xeuo pipefail
+
+dn=$(dirname $0)
+. ${dn}/../../ci/libpaprci/libbuild.sh
+
+pkg_upgrade
+pkg_install git rsync openssh-clients ansible standard-test-roles
+
+# "Hot patch" this to pick up https://pagure.io/standard-test-roles/pull-request/152
+# so we get parallelism
+cd /usr/share/ansible/inventory && curl -L -O https://pagure.io/standard-test-roles/raw/master/f/inventory/standard-inventory-qcow2

tests/installed/run.sh

@@ -1,17 +1,17 @@
-#!/bin/bash
+#!/usr/bin/bash
-
+# Run all installed tests; see README.md in this directory for more
+# information.
 set -xeuo pipefail
 
-dn=$(dirname $0)
+dn=$(cd $(dirname $0) && pwd)
-for tn in ${dn}/itest-*.sh; do
+
-    if [ -n "${TESTS:-}" ]; then
+if ! test -d build; then
-      tbn=$(basename "$tn" .sh)
+    mkdir -p build
-      tbn=" ${tbn#itest-} "
+    (cd build && ${dn}/../../ci/build-rpm.sh)
-      if [[ " $TESTS " != *$tbn* ]]; then
+fi
-        echo "Skipping: ${tn}"
+
-        continue
+# TODO: parallelize this
-      fi
+PLAYBOOKS=${PLAYBOOKS:-nondestructive.yml destructive.yml}
-    fi
+for playbook in $PLAYBOOKS; do
-    echo Executing: ${tn}
+    time ${dn}/playbook-run.sh -v ${dn}/${playbook}
-    ${tn}
 done