ci: Rework installed tests to use Fedora Standard Test interface

Reusing the way `standard-test-roles` has support for booting
a qcow2 actually gets us to the "VM-in-container" flow.  Plus
Ansible over shell script is sometimes nicer.

https://fedoraproject.org/wiki/CI/Tests#Testing_an_Atomic_Host

It's better than what we were doing before for installed tests,
and moreover using Ansible more broadly for testing is going
to align us better with Fedora's CI.

As part of this I split off a "libpaprci" which I intend to maintain
as a "copylib" for a little bit between ostree/rpm-ostree, and then
we'll figure out how to expand from there (maybe some of the patterns
get "baked in" to PAPR for example).

Note the `FAH27-insttests` context moves to the top since it's now
of primary importance, and I expect that we start expanding it.

Closes: #1462
Approved by: jlebon

ci/build-check.sh

@@ -4,7 +4,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 ${dn}/build.sh
 topdir=$(git rev-parse --show-toplevel)
 resultsdir=$(mktemp -d)
@@ -14,9 +14,21 @@ make syntax-check  # TODO: do syntax-check under check
 for x in test-suite.log config.log; do
     mv ${x} ${resultsdir}
 done
-# And now run the installed tests
+# And now install; we'll run the test suite after we do a clang build first
+# (But we don't install that one)
 make install
 
+# And now a clang build to find unused variables because it does a better
+# job than gcc for vars with cleanups; perhaps in the future these could
+# parallelize
+if test -x /usr/bin/clang; then
+    # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument]
+    export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}"
+    git clean -dfx && git submodule foreach git clean -dfx
+    export CC=clang
+    build
+fi
+
 copy_out_gdtr_artifacts() {
     # Keep this in sync with papr.yml
     # TODO; Split the main/clang builds into separate build dirs
@@ -40,14 +52,3 @@ if test -x /usr/bin/gnome-desktop-testing-runner; then
     # Use the new -L option
     gnome-desktop-testing-runner -L ${resultsdir}/gdtr-results -p 0 ${INSTALLED_TESTS_PATTERN:-libostree/}
 fi
-
-# And now a clang build to find unused variables because it does a better
-# job than gcc for vars with cleanups; perhaps in the future these could
-# parallelize
-if test -x /usr/bin/clang; then
-    # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument]
-    export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}"
-    git clean -dfx && git submodule foreach git clean -dfx
-    export CC=clang
-    build
-fi

ci/build-rpm.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/bash
+# Generate a src.rpm, then binary rpms in the current directory
+
+set -xeuo pipefail
+
+dn=$(dirname $0)
+paprcidir=${dn}/libpaprci
+. ${paprcidir}/libbuild.sh
+
+# Auto-provision bootstrap resources if run as root (normally in CI)
+if test "$(id -u)" == 0; then
+    pkg_install_buildroot
+    pkg_install make /usr/bin/rpmbuild git
+fi
+
+# PAPR really should do this
+if ! test -f libglnx/README.md || ! test -f bsdiff/README.md; then
+    git submodule update --init
+fi
+
+# Default libcurl on by default in fedora unless libsoup is enabled
+if test "${OS_ID}" = 'fedora'; then
+    case "${CONFIGOPTS:-}" in
+        *--with-soup*|*--without-curl*) ;;
+        *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl"
+    esac
+fi
+case "${CONFIGOPTS:-}" in
+    *--with-curl*|*--with-soup*)
+        if test -x /usr/bin/gnome-desktop-testing-runner; then
+            CONFIGOPTS="${CONFIGOPTS} --enable-installed-tests=exclusive"
+        fi
+        ;;
+esac
+
+# TODO: Use some form of rpm's --build-in-place to skip archive-then-unpack?
+make -f ${paprcidir}/Makefile.dist-packaging srpm PACKAGE=libostree DISTGIT_NAME=ostree
+if test "$(id -u)" == 0; then
+    pkg_builddep *.src.rpm
+else
+    echo "NOTE: Running as non-root, assuming build dependencies are installed"
+fi
+if ! ${paprcidir}/rpmbuild-cwd --rebuild *.src.rpm; then
+    find . -type f -name config.log -exec cat {} \;
+    exit 1
+fi

ci/build.sh

@@ -4,12 +4,11 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 pkg_upgrade
-pkg_install_builddeps ostree
-# Until this propagates farther
-pkg_install 'pkgconfig(libcurl)' 'pkgconfig(openssl)'
+pkg_install_buildroot
+pkg_builddep ostree
 pkg_install sudo which attr fuse strace \
     libubsan libasan libtsan PyYAML redhat-rpm-config \
     elfutils
@@ -18,15 +17,13 @@ if test -n "${CI_PKGS:-}"; then
 fi
 pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang \
                   python3-PyYAML
-(. /etc/os-release;
- if test "${ID}" = "centos"; then
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-     pkg_install python34{,-PyYAML}
- fi
-)
+if test "${OS_ID}" = "centos"; then
+    rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    pkg_install python34{,-PyYAML}
+fi
 
 # Default libcurl on by default in fedora unless libsoup is enabled
-if sh -c '. /etc/os-release; test "${ID}" = fedora'; then
+if test "${OS_ID}" = 'fedora'; then
     case "${CONFIGOPTS:-}" in
         *--with-soup*|*--without-curl*) ;;
         *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl"

ci/ci-commitmessage-submodules.sh

@@ -17,7 +17,7 @@ set -euo pipefail
 # being tested rather than the merge sha
 HEAD=${PAPR_COMMIT:-HEAD}
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 tmpd=$(mktemp -d)
 touch ${tmpd}/.tmpdir

ci/flatpak.sh

@@ -9,13 +9,13 @@ set -xeuo pipefail
 FLATPAK_TAG=0.10.2.1
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 codedir=$(pwd)
 
 pkg_upgrade
-pkg_install_builddeps ostree
-pkg_install_builddeps flatpak
+pkg_install_buildroot
+pkg_builddep ostree flatpak
 pkg_install gettext-devel # A new dependency
 # Copy of builddeps from build.sh in flatpak
 pkg_install sudo which attr fuse \

ci/libbuild.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/bash
-
-pkg_upgrade() {
-    # https://bugzilla.redhat.com/show_bug.cgi?id=1483553
-    ecode=0
-    yum -y distro-sync 2>err.txt || ecode=$?
-    if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
-        rpm --rebuilddb
-        yum -y distro-sync
-    else
-        if test ${ecode} '!=' 0; then
-            cat err.txt
-            exit ${ecode}
-        fi
-    fi
-}
-
-make() {
-    /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@"
-}
-
-build() {
-    env NOCONFIGURE=1 ./autogen.sh
-    ./configure --prefix=/usr --libdir=/usr/lib64 "$@"
-    make V=1
-}
-
-pkg_install() {
-    yum -y install "$@"
-}
-
-pkg_install_if_os() {
-    os=$1
-    shift
-    (. /etc/os-release;
-         if test "${os}" = "${ID}"; then
-            pkg_install "$@"
-         else
-             echo "Skipping installation on OS ${ID}: $@"
-         fi
-    )
-}
-
-pkg_builddep() {
-    # This is sadly the only case where it's a different command
-    if test -x /usr/bin/dnf; then
-        dnf builddep -y "$@"
-    else
-        yum-builddep -y "$@"
-    fi
-}
-
-pkg_install_builddeps() {
-    pkg=$1
-    if test -x /usr/bin/dnf; then
-        yum -y install dnf-plugins-core
-        yum install -y 'dnf-command(builddep)'
-        # Base buildroot
-        pkg_install @buildsys-build
-    else
-        yum -y install yum-utils
-        # Base buildroot, copied from the mock config sadly
-        yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz
-    fi
-    # builddeps+runtime deps
-    pkg_builddep $pkg
-    pkg_install $pkg
-    rpm -e $pkg
-}

ci/libpaprci/Makefile.dist-packaging

@@ -0,0 +1,39 @@
+# -*- mode: Makefile -*-
+
+mypath = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+topsrcdir = $(shell git rev-parse --show-toplevel)
+GITREV = $(shell git describe --always --tags)
+GITREV_FOR_PKG = $(shell echo "$(GITREV)" | sed -e 's,-,\.,g' -e 's,^v,,')
+
+PACKAGE ?= $(shell basename $(topsrcdir))
+DISTGIT_NAME ?= $(PACKAGE)
+DISTGIT ?= https://src.fedoraproject.org/rpms/$(DISTGIT_NAME)
+SPEC ?= $(topsrcdir)/$(DISTGIT_NAME).spec
+
+PKG_VER = $(PACKAGE)-$(GITREV_FOR_PKG)
+PKG_CLIENT_VER = $(PACKAGE)-client-$(GITREV_FOR_PKG)
+
+dist-snapshot:
+	if ! test -f $(PKG_VER).tar.xz; then \
+	  $(mypath)/make-git-snapshot.sh "$(topsrcdir)" "$(PKG_VER)" "$(GITREV)" && \
+	  rm -f $(PKG_VER).tar.xz && \
+	  xz $(PKG_VER).tar; \
+  fi
+
+srpm: dist-snapshot
+	if test -f "$(SPEC)"; then \
+	   sed -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(SPEC) > $(DISTGIT_NAME).spec && \
+	   $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec ; \
+	else \
+	  test -d $(DISTGIT_NAME) || git clone --depth=1 $(DISTGIT) && \
+    mv $(PKG_VER).tar.xz $(DISTGIT_NAME) && \
+    origdir=$$(pwd); \
+	  cd $(DISTGIT_NAME) && \
+		git stash && git pull -r && \
+	  sed -i -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(DISTGIT_NAME).spec && \
+    rm -f *.src.rpm && \
+	  $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec && mv *.src.rpm $${origdir}; \
+	fi
+
+rpm: srpm
+	./rpmbuild-cwd --rebuild $(PKG_VER)*.src.rpm

ci/libpaprci/libbuild.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/bash
+
+dn=$(cd $(dirname $0) && pwd)
+
+OS_ID=$(. /etc/os-release; echo $ID)
+OS_VERSION_ID=$(. /etc/os-release; echo $VERSION_ID)
+
+pkg_upgrade() {
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1483553
+    local ecode=0
+    yum -y distro-sync 2>err.txt || ecode=$?
+    if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then
+        rpm --rebuilddb
+        yum -y distro-sync
+    else
+        if test ${ecode} '!=' 0; then
+            cat err.txt
+            exit ${ecode}
+        fi
+    fi
+}
+
+make() {
+    /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@"
+}
+
+build() {
+    env NOCONFIGURE=1 ./autogen.sh
+    ./configure --sysconfdir=/etc --prefix=/usr --libdir=/usr/lib64 "$@"
+    make V=1
+}
+
+pkg_install() {
+    yum -y install "$@"
+}
+
+pkg_install_if_os() {
+    local os=$1
+    shift
+    if test "${os}" = "${OS_ID}"; then
+        pkg_install "$@"
+    else
+        echo "Skipping installation targeted for ${os} (current OS: ${OS_ID}): $@"
+    fi
+}
+
+pkg_install_buildroot() {
+    case "${OS_ID}" in
+        fedora) pkg_install dnf-plugins-core @buildsys-build;;
+        centos) pkg_install yum-utils
+                # Base buildroot, copied from the mock config sadly
+                pkg_install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ \
+                            grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar \
+                            unzip util-linux which xz;;
+        *) fatal "pkg_install_buildroot(): Unhandled OS ${OS_ID}";;
+    esac
+}
+
+pkg_builddep() {
+    # This is sadly the only case where it's a different command
+    if test -x /usr/bin/dnf; then
+        dnf builddep -y "$@"
+    else
+        yum-builddep -y "$@"
+    fi
+}
+
+# Install both build and runtime dependencies for $pkg
+pkg_builddep_runtimedep() {
+    local pkg=$1
+    pkg_builddep $pkg
+    pkg_install $pkg
+    rpm -e $pkg
+}

ci/libpaprci/make-git-snapshot.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+srcdir=$1
+shift
+PKG_VER=$1
+shift
+GITREV=$1
+shift
+
+TARFILE=${PKG_VER}.tar
+TARFILE_TMP=${TARFILE}.tmp
+
+set -x
+set -e
+
+test -n "${srcdir}"
+test -n "${PKG_VER}"
+test -n "${GITREV}"
+
+TOP=$(git rev-parse --show-toplevel)
+
+echo "Archiving ${PKG_VER} at ${GITREV} to ${TARFILE_TMP}"
+(cd ${TOP}; git archive --format=tar --prefix=${PKG_VER}/ ${GITREV}) > ${TARFILE_TMP}
+ls -al ${TARFILE_TMP}
+(cd ${TOP}; git submodule status) | while read line; do
+    rev=$(echo ${line} | cut -f 1 -d ' '); path=$(echo ${line} | cut -f 2 -d ' ')
+    echo "Archiving ${path} at ${rev}"
+    (cd ${srcdir}/${path}; git archive --format=tar --prefix=${PKG_VER}/${path}/ ${rev}) > submodule.tar
+    tar -A -f ${TARFILE_TMP} submodule.tar
+    rm submodule.tar
+done
+mv ${TARFILE_TMP} ${TARFILE}

ci/libpaprci/rpmbuild-cwd

@@ -0,0 +1,11 @@
+#!/bin/sh
+# rpmbuild-cwd:
+# Run "rpmbuild", defining all RPM variables to use the current directory.
+# This matches Fedora's system.
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+pwd=$(pwd)
+exec rpmbuild --define "_sourcedir ${pwd}" --define "_specdir ${pwd}" --define "_builddir ${pwd}" --define "_srcrpmdir ${pwd}" --define "_rpmdir ${pwd}" --define "_buildrootdir ${pwd}/.build" "$@"

ci/provision-prep.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/bash
+# Prepare the current environment
+
+set -xeuo pipefail
+
+dn=$(dirname $0)
+. ${dn}/libpaprci/libbuild.sh
+pkg_upgrade
+pkg_install_buildroot
+pkg_install sudo which attr fuse strace \
+            libubsan libasan libtsan PyYAML elfutils
+pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang
+
+if test -n "${CI_PKGS:-}"; then
+    pkg_install ${CI_PKGS}
+fi

ci/rpmostree.sh

@@ -9,13 +9,13 @@ set -xeuo pipefail
 RPMOSTREE_TAG=v2017.11
 
 dn=$(dirname $0)
-. ${dn}/libbuild.sh
+. ${dn}/libpaprci/libbuild.sh
 
 codedir=$(pwd)
 
 pkg_upgrade
-pkg_install_builddeps ostree
-pkg_install_builddeps rpm-ostree
+pkg_install_buildroot
+pkg_builddep ostree rpm-ostree
 pkg_install rpm-ostree && rpm -e rpm-ostree
 
 # Duplicate of deps from build.sh in rpm-ostree for tests