From 92284f9b8107078d93423f772929bed0ab15e75b Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 16 Jul 2020 11:42:57 -0400 Subject: [PATCH 1/2] ci: Constrain parallel build jobs The default `_NPROCESSORS_ONLN` heuristic we have isn't cgroups aware. So it thinks it has e.g. 40 CPUs when running in a k8s pod. This can then blow through our allocated resource limits. Declare some modest amount of RAM and CPU resources and override `make` parallelism. This matches what rpm-ostree now does in https://github.com/coreos/rpm-ostree/pull/2155. --- .cci.jenkinsfile | 12 +++++++----- ci/libbuild.sh | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 4315a1d0..2a4c7288 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -2,7 +2,8 @@ stage("Build") { parallel normal: { - cosaPod(buildroot: true, runAsUser: 0) { + def n = 5 + cosaPod(buildroot: true, runAsUser: 0, memory: "2Gi", cpu: "${n}") { checkout scm stage("Core build") { shwrap(""" @@ -10,7 +11,7 @@ parallel normal: { git fetch origin --tags git submodule update --init - env SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} SKIP_INSTALLDEPS=1 ./ci/build.sh """) } stage("Unit tests") { @@ -84,18 +85,19 @@ parallel fcos: { } }, buildopts: { - cosaPod(buildroot: true, runAsUser: 0) { + def n = 5 + cosaPod(buildroot: true, runAsUser: 0, memory: "2Gi", cpu: "${n}") { checkout scm shwrap(""" git submodule update --init git worktree add build-rust && cd build-rust - env CONFIGOPTS="--enable-rust" SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} CONFIGOPTS="--enable-rust" SKIP_INSTALLDEPS=1 ./ci/build.sh make check TESTS=tests/test-rollsum cd .. && rm -rf build-rust git worktree add build-libsoup && cd build-libsoup - env CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh make check cd .. && rm -rf build-libsoup """) diff --git a/ci/libbuild.sh b/ci/libbuild.sh index a8ade0d7..dece8d09 100644 --- a/ci/libbuild.sh +++ b/ci/libbuild.sh @@ -10,7 +10,7 @@ pkg_upgrade() { } make() { - /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@" + /usr/bin/make -j ${MAKE_JOBS:-$(getconf _NPROCESSORS_ONLN)} "$@" } build() { From 6a5f97c14522f46f202f6cf9449c6e9fe0f4d15b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Jul 2020 01:36:51 +0000 Subject: [PATCH 2/2] tests: Add needs-internet tag for webserver bits Fixes the tests, see https://github.com/coreos/coreos-assembler/pull/1600 TODO: provide a webserver binary via virtio or so --- tests/kolainst/nondestructive/itest-payload-link.sh | 2 ++ tests/kolainst/nondestructive/itest-pull.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kolainst/nondestructive/itest-payload-link.sh b/tests/kolainst/nondestructive/itest-payload-link.sh index 6cfe291a..cbd82d41 100755 --- a/tests/kolainst/nondestructive/itest-payload-link.sh +++ b/tests/kolainst/nondestructive/itest-payload-link.sh @@ -1,4 +1,6 @@ #!/bin/bash +# FIXME just for webserver +# kola: { "tags": "needs-internet" } # # Copyright (C) 2018 Red Hat, Inc. # diff --git a/tests/kolainst/nondestructive/itest-pull.sh b/tests/kolainst/nondestructive/itest-pull.sh index 770f2444..5e8776aa 100755 --- a/tests/kolainst/nondestructive/itest-pull.sh +++ b/tests/kolainst/nondestructive/itest-pull.sh @@ -1,5 +1,6 @@ #!/bin/bash - +# FIXME just for webserver +# kola: { "tags": "needs-internet" } # Using the host ostree, test HTTP pulls set -xeuo pipefail