Add --enable-installed-tests=exclusive, fix installed case

The major reason to do this is that running tests *both* installed
and uninstalled in our CI is a mostly pointless waste of time.
Particularly given we have a few expensive tests.

We *do* have tests that only run uninstalled (since they require
the source code) like `test-symbols.sh`.

Hence, add `--enable-installed-tests=exclusive` to mean *only* do installed for
most tests.

We'll still have uninstalled coverage via the Travis/Debian configs, and we
could perhaps do another build with a subset of uninstalled tests, but I'm not
really concerned about it.

I'd like to do a renewed push for the InstalledTests model since
I feel it's just fundamentally better.  (`g-d-t-r` kind of sucks,
but then so does the automake runner).

Also while we're here - fix the CI to use the correct context,
which started this mess.

Closes: #837
Approved by: dbnicholson

.redhat-ci.yml

@@ -23,14 +23,16 @@ build:
     config-opts: >
       --prefix=/usr
       --libdir=/usr/lib64
-      --enable-installed-tests
+      --enable-installed-tests=exclusive
       --enable-gtk-doc
 
+# The g-d-t-r timeout is for test-pull-many.sh; if tweaking this,
+# also be sure to change the other cases below
 tests:
     - make syntax-check
     - ./tests/ci-commitmessage-submodules.sh
     - make check
-    - gnome-desktop-testing-runner -p 0 ostree
+    - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/'
 
 timeout: 30m
 
@@ -65,7 +67,6 @@ build:
     config-opts: >
       --prefix=/usr
       --libdir=/usr/lib64
-      --enable-installed-tests
       --enable-gtk-doc
       --enable-rust
 
@@ -90,14 +91,14 @@ build:
     config-opts: >
       --prefix=/usr
       --libdir=/usr/lib64
-      --enable-installed-tests
+      --enable-installed-tests=exclusive
       --enable-gtk-doc
       --with-curl
       --with-openssl
 
 tests:
   - make check
-  - gnome-desktop-testing-runner -p 0 ostree
+  - /bin/sh -c 'gnome-desktop-testing-runner -p 0 --timeout $((10 * 60)) libostree/'
 
 artifacts:
   - test-suite.log

Makefile-tests.am

@@ -42,9 +42,16 @@ endif
 uninstalled_test_data = tests/ostree-symlink-stamp tests/ostree-prepare-root-symlink-stamp \
 			tests/ostree-remount-symlink-stamp tests/rofiles-fuse-symlink-stamp
 
-dist_uninstalled_test_scripts = tests/test-symbols.sh
+dist_uninstalled_test_scripts = tests/test-symbols.sh tests/coccinelle.sh
 
-dist_test_scripts = \
+# This logic implements ENABLE_INSTALLED_TESTS_EXCLUSIVE; see below.
+# The goal here if installed tests are enabled, we explicitly make the
+# tests *only* run installed, to avoid having to run them twice in CI.
+# This overrides the glib-tap.mk emphasis on doing both, if we'd
+# used e.g. `dist_test_scripts`.
+dist_test_scripts = $(NULL)
+test_programs = $(NULL)
+_installed_or_uninstalled_test_scripts = \
 	tests/test-basic.sh \
 	tests/test-basic-user.sh \
 	tests/test-basic-user-only.sh \
@@ -102,23 +109,22 @@ dist_test_scripts = \
 	tests/test-pull-contenturl.sh \
 	tests/test-pull-mirrorlist.sh \
 	tests/test-summary-view.sh \
-	tests/coccinelle.sh \
 	$(NULL)
 
 if BUILDOPT_FUSE
-dist_test_scripts += tests/test-rofiles-fuse.sh
+_installed_or_uninstalled_test_scripts += tests/test-rofiles-fuse.sh
 else
 EXTRA_DIST += tests/test-rofiles-fuse.sh
 endif
 
 if USE_LIBSOUP
-dist_test_scripts += tests/test-remote-cookies.sh
+_installed_or_uninstalled_test_scripts += tests/test-remote-cookies.sh
 endif
 
 # These call into gjs scripts
 js_tests = tests/test-corruption.sh tests/test-pull-corruption.sh
 if BUILDOPT_GJS
-dist_test_scripts += $(js_tests)
+_installed_or_uninstalled_test_scripts += $(js_tests)
 else
 EXTRA_DIST += $(js_tests)
 endif
@@ -182,7 +188,7 @@ if !ENABLE_INSTALLED_TESTS
 libreaddir_rand_la_LDFLAGS += -rpath $(abs_builddir)
 endif
 
-test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \
+_installed_or_uninstalled_test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \
 	tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \
 	tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \
 	tests/test-basic-c tests/test-sysroot-c tests/test-pull-c
@@ -191,7 +197,7 @@ test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tes
 noinst_PROGRAMS += tests/test-rollsum-cli
 
 if USE_LIBARCHIVE
-test_programs += tests/test-libarchive-import
+_installed_or_uninstalled_test_programs += tests/test-libarchive-import
 endif
 
 common_tests_cflags = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx
@@ -288,6 +294,17 @@ tests/%-symlink-stamp: % Makefile
 	ln -sf "$${real_bin}" tests/$*; \
 	touch $@
 
+# See above comment on binding the tests to be either installed or not.
+if ENABLE_INSTALLED_TESTS_EXCLUSIVE
+dist_installed_test_scripts = $(_installed_or_uninstalled_test_scripts)
+installed_test_programs = $(_installed_or_uninstalled_test_programs)
+check-local:
+	echo "NOTE: Exclusive installed tests are enabled; to run them, make install, then: gnome-desktop-testing-runner -p 0 libostree/"
+else
+dist_test_scripts += $(_installed_or_uninstalled_test_scripts)
+test_programs += $(_installed_or_uninstalled_test_programs)
+endif
+
 # Unfortunately the glib test data APIs don't actually handle
 # non-recursive Automake, so we change our code to canonically look
 # for tests/ which is just a symlink when installed.

buildutil/glibtests.m4

@@ -1,17 +1,21 @@
 dnl GLIB_TESTS
-dnl
+dnl NOTE: this file has been modified from upstream glib; see
+dnl https://github.com/ostreedev/ostree/pull/837
 
 AC_DEFUN([GLIB_TESTS],
 [
   AC_ARG_ENABLE(installed-tests,
                 AS_HELP_STRING([--enable-installed-tests],
                                [Enable installation of some test cases]),
-                [case ${enableval} in
+                [enable_installed_tests=${enableval};
+                 case ${enableval} in
                   yes) ENABLE_INSTALLED_TESTS="1"  ;;
+                  exclusive) ENABLE_INSTALLED_TESTS="1"; ENABLE_INSTALLED_TESTS_EXCLUSIVE=1 ;;
                   no)  ENABLE_INSTALLED_TESTS="" ;;
                   *) AC_MSG_ERROR([bad value ${enableval} for --enable-installed-tests]) ;;
                  esac])
   AM_CONDITIONAL([ENABLE_INSTALLED_TESTS], test "$ENABLE_INSTALLED_TESTS" = "1")
+  AM_CONDITIONAL([ENABLE_INSTALLED_TESTS_EXCLUSIVE], test "$ENABLE_INSTALLED_TESTS_EXCLUSIVE" = "1")
   AC_ARG_ENABLE(always-build-tests,
                 AS_HELP_STRING([--enable-always-build-tests],
                                [Enable always building tests during 'make all']),

configure.ac

@@ -457,6 +457,7 @@ echo "
     wrpseudo-compat:                              $enable_wrpseudo_compat
     man pages (xsltproc):                         $enable_man
     api docs (gtk-doc):                           $enable_gtk_doc
+    installed tests:                              $enable_installed_tests
     gjs-based tests:                              $have_gjs
     dracut:                                       $with_dracut
     mkinitcpio:                                   $with_mkinitcpio

tests/ci-build.sh

@@ -85,7 +85,6 @@ make="make -j${ci_parallel} V=1 VERBOSE=1"
 
 ../configure \
     --enable-always-build-tests \
-    --enable-installed-tests \
     ${ci_configopts}
     "$@"
 
@@ -106,12 +105,6 @@ if [ "$ci_sudo" = yes ] && [ "$ci_test" = yes ]; then
         ${make} installcheck || \
     maybe_fail_tests
     cat test/test-suite.log || :
-
-    env \
-        LD_LIBRARY_PATH=/usr/local/lib \
-        GI_TYPELIB_PATH=/usr/local/lib/girepository-1.0 \
-        gnome-desktop-testing-runner -d /usr/local/share ostree/ || \
-    maybe_fail_tests
 fi
 
 # vim:set sw=4 sts=4 et: