From 9ff141fe80f145862c947c3a25d9367c01ebe43c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 18 Jun 2021 11:00:16 -0600 Subject: [PATCH 1/3] ci: Rename GitHub Actions rust workflow metadata file This workflow is specific to using rust and not just the general test suite. --- .github/workflows/{tests.yml => rust.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{tests.yml => rust.yml} (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/rust.yml similarity index 100% rename from .github/workflows/tests.yml rename to .github/workflows/rust.yml From a0012ae0eea82d47a83b44a938d621535b7bc2da Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 18 Jun 2021 11:33:56 -0600 Subject: [PATCH 2/3] ci: Add GitHub Actions workflow for test suite This runs the test suite in various distros. The intention is to use this to replace the Travis CI setup since it often has rate limit failures. Each configuration in the matrix runs in a Docker container, installs system dependencies and then builds and tests ostree. The scripts are basically copy and paste of the travis ones with some of the lesser used features pruned out. Some differences from the travis setup: * OS details are gathered from `/etc/os-release` instead of being passed in as environment variables. * The scripts always assume the user is root and don't try to use `sudo`. * The `installcheck` test has been removed since ostree doesn't actually use that. It could be added to run the installed tests or `gnome-desktop-testing-runner` could just be called directly. There should be enough flexibility to run other distros like Fedora, Arch or Alpine. Another option would be to use the other build scripts in ci/. --- .github/workflows/tests.yml | 106 ++++++++++++++++++++++++++++++++++++ ci/gh-build.sh | 62 +++++++++++++++++++++ ci/gh-install.sh | 84 ++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100755 ci/gh-build.sh create mode 100755 ci/gh-install.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..436dfbe9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,106 @@ +--- +name: Tests +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + tests: + # Distro configuration matrix + # + # Each build is run in a Docker container specific to the distro. + # When adding a new distro, handle the dependency installation in + # `ci/gh-install.sh`. The matrix configuration options are: + # + # name: A friendly name to use for the job. + # + # image: The Docker image to use. + # + # pre-checkout-setup: Commands to run before the git repo checkout. + # If git is not in the Docker image, it must be installed here. + # Otherwise, the checkout action uses the GitHub REST API, which + # doesn't result in an actual git repo. A real git repo is + # required to checkout the submodules. + # + # extra-packages: Packages to install in addition to those in + # `ci/gh-install.sh`. This can be used to support features from + # additional `configure` options. + # + # configure-options: Options to pass to `configure`. + strategy: + matrix: + include: + - name: Debian Buster (10) with sign-ed25519 + image: debian:buster-slim + pre-checkout-setup: | + apt-get update + apt-get install -y git + extra-packages: >- + libsodium-dev + configure-options: >- + --with-ed25519-libsodium + + - name: Debian Buster (10) with curl, sign-ed25519 and no gpgme + image: debian:buster-slim + pre-checkout-setup: | + apt-get update + apt-get install -y git + extra-packages: >- + libsodium-dev + configure-options: >- + --with-curl + --with-ed25519-libsodium + --without-gpgme + + # A 32 bit build to act as a proxy for frequently deployed 32 + # bit armv7 + - name: Debian Buster (10) 32 bit + image: i386/debian:buster-slim + # This is pretty nasty. The checkout action uses an x86_64 + # node binary in the container, so we need to provide an + # x86_64 ld.so and libstdc++. + pre-checkout-setup: | + dpkg --add-architecture amd64 + apt-get update + apt-get install -y git libc6:amd64 libstdc++6:amd64 + + - name: Ubuntu Focal (20.04) + image: ubuntu:focal + pre-checkout-setup: | + apt-get update + apt-get install -y git + + - name: Ubuntu Groovy (20.10) + image: ubuntu:groovy + pre-checkout-setup: | + apt-get update + apt-get install -y git + + name: ${{ matrix.name }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.image }} + + steps: + - name: Pre-checkout setup + run: ${{ matrix.pre-checkout-setup }} + if: ${{ matrix.pre-checkout-setup }} + + - name: Checkout repository + uses: actions/checkout@v2 + with: + submodules: true + + - name: Install dependencies + run: ./ci/gh-install.sh ${{ matrix.extra-packages }} + + - name: Build and test + run: ./ci/gh-build.sh ${{ matrix.configure-options }} + env: + # GitHub hosted runners currently have 2 CPUs, so run 2 + # parallel make jobs. + # + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners + MAKEFLAGS: -j2 diff --git a/ci/gh-build.sh b/ci/gh-build.sh new file mode 100755 index 00000000..949ce5fa --- /dev/null +++ b/ci/gh-build.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright © 2015-2016 Collabora Ltd. +# Copyright © 2021 Endless OS Foundation LLC +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail +set -x + +NOCONFIGURE=1 ./autogen.sh + +srcdir="$(pwd)" +mkdir ci-build +cd ci-build + +make="make V=1 VERBOSE=1" + +../configure \ + --enable-always-build-tests \ + "$@" + +${make} + +# Run the tests both using check and distcheck and dump the logs on +# failures. For distcheck the logs will be inside the dist directory, so +# tell make to use the current directory. +if ! ${make} check; then + cat test-suite.log || : + exit 1 +fi +if ! ${make} distcheck \ + TEST_SUITE_LOG=$(pwd)/test-suite.log \ + DISTCHECK_CONFIGURE_FLAGS="$*" +then + cat test-suite.log || : + exit 1 +fi + +# Show the installed files +${make} install DESTDIR=$(pwd)/DESTDIR +( cd DESTDIR && find . ) + +# vim:set sw=4 sts=4 et: diff --git a/ci/gh-install.sh b/ci/gh-install.sh new file mode 100755 index 00000000..dc0c7fde --- /dev/null +++ b/ci/gh-install.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright © 2015-2016 Collabora Ltd. +# Copyright © 2021 Endless OS Foundation LLC +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail +set -x + +NULL= + +# Get the OS release info +. /etc/os-release + +case "$ID" in + (debian|ubuntu) + # Make debconf run non-interactively since its questions can't + # be answered. + export DEBIAN_FRONTEND=noninteractive + + # TODO: fetch this list from the Debian packaging git repository? + apt-get -y update + apt-get -y install \ + attr \ + bison \ + cpio \ + debhelper \ + dh-autoreconf \ + dh-systemd \ + docbook-xml \ + docbook-xsl \ + e2fslibs-dev \ + elfutils \ + fuse \ + gjs \ + gnome-desktop-testing \ + gobject-introspection \ + gtk-doc-tools \ + libarchive-dev \ + libattr1-dev \ + libcap-dev \ + libcurl4-openssl-dev \ + libfuse-dev \ + libgirepository1.0-dev \ + libglib2.0-dev \ + libgpgme11-dev \ + liblzma-dev \ + libmount-dev \ + libselinux1-dev \ + libsoup2.4-dev \ + libsystemd-dev \ + procps \ + python3-yaml \ + systemd \ + zlib1g-dev \ + "$@" + ;; + + (*) + echo "Don't know how to set up ${ID}" >&2 + exit 1 + ;; +esac + +# vim:set sw=4 sts=4 et: From 00f7c88a2dce019b9e6f06f31c9e64793f7efe57 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Fri, 18 Jun 2021 15:22:39 -0600 Subject: [PATCH 3/3] ci: So long, Travis CI travis-ci.org stopped running builds on June 15, 2021. Since this organization is very unlikely to switch to travis-ci.com, just drop the setup. The new GitHub Actions tests completely replace it. --- .travis.yml | 18 ----- ci/travis-Dockerfile.in | 10 --- ci/travis-build.sh | 110 ------------------------------- ci/travis-install.sh | 141 ---------------------------------------- 4 files changed, 279 deletions(-) delete mode 100644 .travis.yml delete mode 100644 ci/travis-Dockerfile.in delete mode 100755 ci/travis-build.sh delete mode 100755 ci/travis-install.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 10d8e4bf..00000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: c -sudo: required - -env: - # debian has libsodium-dev, ubuntu doesn't in core at least - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=buster ci_configopts="--with-ed25519-libsodium" ci_pkgs="libsodium-dev" - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=buster ci_configopts="--with-curl --with-ed25519-libsodium --without-gpgme" ci_pkgs="libsodium-dev" - # A 32 bit build to act as a proxy for frequently deployed 32 bit armv7 - - ci_docker=i386/debian:buster-slim ci_distro=debian ci_suite=buster - - ci_docker=ubuntu:focal ci_distro=ubuntu ci_suite=focal - - ci_docker=ubuntu:groovy ci_distro=ubuntu ci_suite=groovy - -script: - - ci/travis-install.sh - - ci_parallel=2 ci_sudo=yes ci/travis-build.sh - -notifications: - email: false diff --git a/ci/travis-Dockerfile.in b/ci/travis-Dockerfile.in deleted file mode 100644 index 9e47876e..00000000 --- a/ci/travis-Dockerfile.in +++ /dev/null @@ -1,10 +0,0 @@ -FROM @ci_docker@ -ENV container docker - -ADD ci/travis-install.sh /travis-install.sh -RUN ci_suite="@ci_suite@" ci_distro="@ci_distro@" ci_pkgs="@ci_pkgs@" ci_in_docker=yes /travis-install.sh - -ADD . /home/user/ostree -RUN chown -R user:user /home/user/ostree -WORKDIR /home/user/ostree -USER user diff --git a/ci/travis-build.sh b/ci/travis-build.sh deleted file mode 100755 index e31da81f..00000000 --- a/ci/travis-build.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -# Copyright © 2015-2016 Collabora Ltd. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -set -euo pipefail -set -x - -NULL= - -# ci_docker: -# If non-empty, this is the name of a Docker image. travis-install.sh will -# fetch it with "docker pull" and use it as a base for a new Docker image -# named "ci-image" in which we will do our testing. -# -# If empty, we test on "bare metal". -# Typical values: ubuntu:xenial, debian:jessie-slim -: "${ci_docker:=}" - -# ci_parallel: -# A number of parallel jobs, passed to make -j -: "${ci_parallel:=1}" - -# ci_sudo: -# If yes, assume we can get root using sudo; if no, only use current user -: "${ci_sudo:=no}" - -# ci_test: -# If yes, run tests; if no, just build -: "${ci_test:=yes}" - -# ci_test_fatal: -# If yes, test failures break the build; if no, they are reported but ignored -: "${ci_test_fatal:=yes}" - -# ci_configopts: -# Additional args for configure -: "${ci_configopts:=}" - -if [ -n "$ci_docker" ]; then - exec docker run \ - --env=ci_docker="" \ - --env=ci_parallel="${ci_parallel}" \ - --env=ci_sudo=yes \ - --env=ci_test="${ci_test}" \ - --env=ci_test_fatal="${ci_test_fatal}" \ - --env=ci_configopts="${ci_configopts}" \ - --privileged \ - ci-image \ - ci/travis-build.sh -fi - -maybe_fail_tests () { - if [ "$ci_test_fatal" = yes ]; then - exit 1 - fi -} - -NOCONFIGURE=1 ./autogen.sh - -srcdir="$(pwd)" -mkdir ci-build -cd ci-build - -make="make -j${ci_parallel} V=1 VERBOSE=1" - -../configure \ - --enable-always-build-tests \ - ${ci_configopts} \ - "$@" - -${make} -[ "$ci_test" = no ] || ${make} check || maybe_fail_tests -cat test-suite.log || : -[ "$ci_test" = no ] || ${make} distcheck DISTCHECK_CONFIGURE_FLAGS="${ci_configopts}" || maybe_fail_tests -cat test-suite.log || : - -${make} install DESTDIR=$(pwd)/DESTDIR -( cd DESTDIR && find . ) - -if [ "$ci_sudo" = yes ] && [ "$ci_test" = yes ]; then - sudo ${make} install - env \ - LD_LIBRARY_PATH=/usr/local/lib \ - GI_TYPELIB_PATH=/usr/local/lib/girepository-1.0 \ - ${make} installcheck || \ - maybe_fail_tests - cat test-suite.log || : -fi - -# vim:set sw=4 sts=4 et: diff --git a/ci/travis-install.sh b/ci/travis-install.sh deleted file mode 100755 index b40f4c55..00000000 --- a/ci/travis-install.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# Copyright © 2015-2016 Collabora Ltd. -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -set -euo pipefail -set -x - -NULL= - -# ci_distro: -# OS distribution in which we are testing -# Typical values: ubuntu, debian; maybe fedora in future -: "${ci_distro:=debian}" - -# ci_docker: -# If non-empty, this is the name of a Docker image. travis-install.sh will -# fetch it with "docker pull" and use it as a base for a new Docker image -# named "ci-image" in which we will do our testing. -: "${ci_docker:=}" - -# ci_in_docker: -# Used internally by travis-install.sh. If yes, we are inside the Docker image -# (ci_docker is empty in this case). -: "${ci_in_docker:=no}" - -# ci_suite: -# OS suite (release, branch) in which we are testing. -# Typical values for ci_distro=ubuntu: xenial, trusty -# Typical values for ci_distro=debian: sid, jessie -# Typical values for ci_distro=fedora might be 25, rawhide -: "${ci_suite:=stretch}" - -# ci_configopts: Additional arguments for configure -: "${ci_configopts:=}" - -# ci_pkgs: Additional packages to be installed -: "${ci_pkgs:=}" - -if [ $(id -u) = 0 ]; then - sudo= -else - sudo=sudo -fi - -if [ -n "$ci_docker" ]; then - sed \ - -e "s,@ci_distro@,${ci_distro}," \ - -e "s,@ci_docker@,${ci_docker}," \ - -e "s,@ci_suite@,${ci_suite}," \ - -e "s,@ci_pkgs@,${ci_pkgs}," \ - < ci/travis-Dockerfile.in > Dockerfile - exec docker build -t ci-image . -fi - -case "$ci_distro" in - (debian) - # Docker images use httpredir.debian.org but it seems to be - # unreliable; use a CDN instead - sed -i -e 's/httpredir\.debian\.org/deb.debian.org/g' /etc/apt/sources.list - ;; -esac - -case "$ci_distro" in - (debian|ubuntu) - # Make debconf run non-interactively since its questions can't - # be answered. - export DEBIAN_FRONTEND=noninteractive - - # TODO: fetch this list from the Debian packaging git repository? - $sudo apt-get -y update - $sudo apt-get -y install \ - attr \ - bison \ - cpio \ - debhelper \ - dh-autoreconf \ - dh-systemd \ - docbook-xml \ - docbook-xsl \ - e2fslibs-dev \ - elfutils \ - fuse \ - gjs \ - gnome-desktop-testing \ - gobject-introspection \ - gtk-doc-tools \ - libarchive-dev \ - libattr1-dev \ - libcap-dev \ - libfuse-dev \ - libgirepository1.0-dev \ - libglib2.0-dev \ - libgpgme11-dev \ - liblzma-dev \ - libmount-dev \ - libselinux1-dev \ - libsoup2.4-dev \ - libcurl4-openssl-dev \ - procps \ - zlib1g-dev \ - python3-yaml \ - ${ci_pkgs:-} \ - ${NULL} - - if [ "$ci_in_docker" = yes ]; then - # Add the user that we will use to do the build inside the - # Docker container, and let them use sudo - adduser --disabled-password user /etc/sudoers.d/nopasswd - chmod 0440 /etc/sudoers.d/nopasswd - fi - ;; - - (*) - echo "Don't know how to set up ${ci_distro}" >&2 - exit 1 - ;; -esac - -# vim:set sw=4 sts=4 et: