From 779df11c9ed589989ffa0277434a0f059830e0e6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 15 Feb 2018 08:55:40 -0500 Subject: [PATCH 01/51] build-sys: Post-release version bump Closes: #1455 Approved by: jlebon --- configure.ac | 4 ++-- src/libostree/libostree-devel.sym | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 1b299d55..3b2ff26f 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2018]) -m4_define([release_version], [2]) +m4_define([release_version], [3]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 46ad280c..3f3a45b5 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,6 +18,8 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2018.3 { +} LIBOSTREE_2018.2; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste From 9933de232304f47e2be09c72906cc5183d9bbfda Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 17 Jan 2018 14:25:26 +0000 Subject: [PATCH 02/51] test-concurrency: Use Python 3 syntax for octal This also works in Python 2.7, and is a little clearer. Signed-off-by: Simon McVittie Closes: #1457 Approved by: cgwalters --- tests/test-concurrency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-concurrency.py b/tests/test-concurrency.py index bdcc1d91..6fade24f 100755 --- a/tests/test-concurrency.py +++ b/tests/test-concurrency.py @@ -33,7 +33,7 @@ def fatal(msg): # different files with different checksums. def mktree(dname, serial=0): print('Creating tree', dname, file=sys.stderr) - os.mkdir(dname, 0755) + os.mkdir(dname, 0o755) for v in xrange(20): with open('{}/{}'.format(dname, v), 'w') as f: f.write('{} {} {}\n'.format(dname, serial, v)) From 02dc5e2dd47d1be0eb73d2f7e5b588f04c572390 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 17 Jan 2018 14:42:20 +0000 Subject: [PATCH 03/51] test-concurrency: Replace range with xrange range in Python 3 does what xrange did in Python 2. This still works in Python 2, it just uses a bit more memory. Signed-off-by: Simon McVittie Closes: #1457 Approved by: cgwalters --- tests/test-concurrency.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test-concurrency.py b/tests/test-concurrency.py index 6fade24f..f16f696b 100755 --- a/tests/test-concurrency.py +++ b/tests/test-concurrency.py @@ -34,7 +34,7 @@ def fatal(msg): def mktree(dname, serial=0): print('Creating tree', dname, file=sys.stderr) os.mkdir(dname, 0o755) - for v in xrange(20): + for v in range(20): with open('{}/{}'.format(dname, v), 'w') as f: f.write('{} {} {}\n'.format(dname, serial, v)) @@ -79,12 +79,12 @@ def run(n_committers, n_pruners): print('n_committers', n_committers, 'n_pruners', n_pruners, file=sys.stderr) n_trees = n_committers / 2 - for v in xrange(n_trees): + for v in range(n_trees): mktree('tree{}'.format(v)) - for v in xrange(n_committers): + for v in range(n_committers): committers.add(commit(v / 2)) - for v in xrange(n_pruners): + for v in range(n_pruners): pruners.add(prune()) failed = False @@ -97,7 +97,7 @@ def run(n_committers, n_pruners): if failed: fatal('A child process exited abnormally') - for v in xrange(n_trees): + for v in range(n_trees): shutil.rmtree('tree{}'.format(v)) # No concurrent pruning From 98b597b465a556f3ad5b16478d7edee3037fbd4d Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 17 Jan 2018 15:03:59 +0000 Subject: [PATCH 04/51] test-concurrency: Explicitly use floor division Python 3 is pickier about this. Python 2.7 has Python 3-compatible semantics for division when the division feature is imported from the future. Signed-off-by: Simon McVittie Closes: #1457 Approved by: cgwalters --- tests/test-concurrency.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test-concurrency.py b/tests/test-concurrency.py index f16f696b..3a0ce103 100755 --- a/tests/test-concurrency.py +++ b/tests/test-concurrency.py @@ -17,6 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. +from __future__ import division from __future__ import print_function import os import sys @@ -78,12 +79,12 @@ def run(n_committers, n_pruners): pruners = set() print('n_committers', n_committers, 'n_pruners', n_pruners, file=sys.stderr) - n_trees = n_committers / 2 + n_trees = n_committers // 2 for v in range(n_trees): mktree('tree{}'.format(v)) for v in range(n_committers): - committers.add(commit(v / 2)) + committers.add(commit(v // 2)) for v in range(n_pruners): pruners.add(prune()) @@ -101,8 +102,8 @@ def run(n_committers, n_pruners): shutil.rmtree('tree{}'.format(v)) # No concurrent pruning -run(cpu_count()/2 + 2, 0) +run(cpu_count() // 2 + 2, 0) print("ok no concurrent prunes") -run(cpu_count()/2 + 4, 3) +run(cpu_count() // 2 + 4, 3) print("ok concurrent prunes") From 6cabeaed3f45c8f3e05fb5e1152bfa5b96a3bc79 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 17 Jan 2018 15:19:12 +0000 Subject: [PATCH 05/51] tests/bootloader-entries-crosscheck: Use Python 3-friendly sorting This is a little clearer than a strcmp()-style negative/zero/positive return, and also works in Python 2. Signed-off-by: Simon McVittie Closes: #1457 Approved by: cgwalters --- tests/bootloader-entries-crosscheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/bootloader-entries-crosscheck.py b/tests/bootloader-entries-crosscheck.py index 38e8e451..5faa5487 100755 --- a/tests/bootloader-entries-crosscheck.py +++ b/tests/bootloader-entries-crosscheck.py @@ -38,8 +38,8 @@ def fatal(msg): sys.stderr.write('\n') sys.exit(1) -def compare_entries_descending(a, b): - return int(b['version']) - int(a['version']) +def entry_get_version(entry): + return int(entry['version']) def get_ostree_option(optionstring): for o in optionstring.split(): @@ -65,7 +65,7 @@ for fname in os.listdir(loaderpath): v = line[s+1:] entry[k] = v entries.append(entry) - entries.sort(compare_entries_descending) + entries.sort(key=entry_get_version, reverse=True) # Parse SYSLINUX config with open(syslinuxpath) as f: From 971265e4e48f2ff512633b27c90569315ff168ff Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Wed, 17 Jan 2018 14:07:32 +0000 Subject: [PATCH 06/51] Use Python 3 for tests Signed-off-by: Simon McVittie Closes: #1463 Approved by: cgwalters --- ci/build.sh | 9 ++++++++- ci/travis-install.sh | 2 +- tests/bootloader-entries-crosscheck.py | 2 +- tests/test-basic-user-only.sh | 2 +- tests/test-concurrency.py | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ci/build.sh b/ci/build.sh index b56f57c6..be05604a 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -16,7 +16,14 @@ pkg_install sudo which attr fuse strace \ if test -n "${CI_PKGS:-}"; then pkg_install ${CI_PKGS} fi -pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang +pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang \ + python3-PyYAML +(. /etc/os-release; + if test "${ID}" = "centos"; then + rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + pkg_install python34{,-PyYAML} + fi +) # Default libcurl on by default in fedora unless libsoup is enabled if sh -c '. /etc/os-release; test "${ID}" = fedora'; then diff --git a/ci/travis-install.sh b/ci/travis-install.sh index 5d3d5a91..8dd373d3 100755 --- a/ci/travis-install.sh +++ b/ci/travis-install.sh @@ -110,7 +110,7 @@ case "$ci_distro" in libcurl4-openssl-dev \ procps \ zlib1g-dev \ - python-yaml \ + python3-yaml \ ${NULL} if [ "$ci_in_docker" = yes ]; then diff --git a/tests/bootloader-entries-crosscheck.py b/tests/bootloader-entries-crosscheck.py index 5faa5487..41f6956e 100755 --- a/tests/bootloader-entries-crosscheck.py +++ b/tests/bootloader-entries-crosscheck.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright (C) 2015 Red Hat # diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index bea7b77f..5f27014e 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -28,7 +28,7 @@ extra_basic_tests=5 . $(dirname $0)/basic-test.sh $CMD_PREFIX ostree --version > version.yaml -python -c 'import yaml; yaml.safe_load(open("version.yaml"))' +python3 -c 'import yaml; yaml.safe_load(open("version.yaml"))' echo "ok yaml version" # Reset things so we don't inherit a lot of state from earlier tests diff --git a/tests/test-concurrency.py b/tests/test-concurrency.py index 3a0ce103..3ec3681c 100755 --- a/tests/test-concurrency.py +++ b/tests/test-concurrency.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2017 Colin Walters # From 2f5a34bed98475b966de3c41bcd0275dc4bb4388 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Feb 2018 12:26:32 -0500 Subject: [PATCH 07/51] sysroot: Bump mtime when writing an origin file This ensures that e.g. `rpm-ostreed` will get notified of the changes. Closes: #1464 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 4284b5ae..8fdf28a3 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -847,7 +847,9 @@ merge_configuration (OstreeSysroot *sysroot, return TRUE; } -/* Write the origin file for a deployment. */ +/* Write the origin file for a deployment; this does not bump the mtime, under + * the assumption the caller may be writing multiple. + */ static gboolean write_origin_file_internal (OstreeSysroot *sysroot, OstreeDeployment *deployment, @@ -903,9 +905,15 @@ ostree_sysroot_write_origin_file (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - return write_origin_file_internal (sysroot, deployment, new_origin, - GLNX_FILE_REPLACE_DATASYNC_NEW, - cancellable, error); + if (!write_origin_file_internal (sysroot, deployment, new_origin, + GLNX_FILE_REPLACE_DATASYNC_NEW, + cancellable, error)) + return FALSE; + + if (!_ostree_sysroot_bump_mtime (sysroot, error)) + return FALSE; + + return TRUE; } typedef struct { From c40a47e965d139cf6e72192bd582098a635b72f4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Feb 2018 14:23:38 -0500 Subject: [PATCH 08/51] sysroot: Add API to clean up transient keys in origin files The `origin/unlocked` and `origin/override-commit` keys are examples of state that's really transient; we don't want to maintain them across upgrades. Right now there are bits for this in both `ostree admin upgrade` as well as in rpm-ostree. This new API will slightly clean up both cases, but it's really prep for adding a concept of deployment "pinning" that will live in the new `libostree-transient` group. Closes: #1464 Approved by: jlebon --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-deployment.c | 29 +++++++++++++++++++++++++++ src/libostree/ostree-deployment.h | 15 ++++++++++++++ src/ostree/ot-admin-builtin-upgrade.c | 23 ++++----------------- 5 files changed, 50 insertions(+), 19 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 4421ed10..892aa0c0 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -173,6 +173,7 @@ ostree_deployment_set_index ostree_deployment_set_bootserial ostree_deployment_set_bootconfig ostree_deployment_set_origin +ostree_deployment_origin_remove_transient_state ostree_deployment_clone ostree_deployment_unlocked_state_to_string diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 3f3a45b5..123627a4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -19,6 +19,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.3 { + ostree_deployment_origin_remove_transient_state; } LIBOSTREE_2018.2; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-deployment.c b/src/libostree/ostree-deployment.c index 6431aa96..2c479fdb 100644 --- a/src/libostree/ostree-deployment.c +++ b/src/libostree/ostree-deployment.c @@ -121,6 +121,35 @@ ostree_deployment_set_origin (OstreeDeployment *self, GKeyFile *origin) self->origin = g_key_file_ref (origin); } +/** + * ostree_deployment_origin_remove_transient_state: + * @origin: An origin + * + * The intention of an origin file is primarily describe the "inputs" that + * resulted in a deployment, and it's commonly used to derive the new state. For + * example, a key value (in pure libostree mode) is the "refspec". However, + * libostree (or other applications) may want to store "transient" state that + * should not be carried across upgrades. + * + * This function just removes all members of the `libostree-transient` group. + * The name of that group is available to all libostree users; best practice + * would be to prefix values underneath there with a short identifier for your + * software. + * + * Additionally, this function will remove the `origin/unlocked` and + * `origin/override-commit` members; these should be considered transient state + * that should have been under an explicit group. + * + * Since: 2018.3 + */ +void +ostree_deployment_origin_remove_transient_state (GKeyFile *origin) +{ + g_key_file_remove_group (origin, OSTREE_ORIGIN_TRANSIENT_GROUP, NULL); + g_key_file_remove_key (origin, "origin", "override-commit", NULL); + g_key_file_remove_key (origin, "origin", "unlocked", NULL); +} + void _ostree_deployment_set_bootcsum (OstreeDeployment *self, const char *bootcsum) diff --git a/src/libostree/ostree-deployment.h b/src/libostree/ostree-deployment.h index b4368f46..985c8133 100644 --- a/src/libostree/ostree-deployment.h +++ b/src/libostree/ostree-deployment.h @@ -27,6 +27,17 @@ G_BEGIN_DECLS #define OSTREE_DEPLOYMENT(inst) (G_TYPE_CHECK_INSTANCE_CAST ((inst), OSTREE_TYPE_DEPLOYMENT, OstreeDeployment)) #define OSTREE_IS_DEPLOYMENT(inst) (G_TYPE_CHECK_INSTANCE_TYPE ((inst), OSTREE_TYPE_DEPLOYMENT)) +/** + * OSTREE_ORIGIN_TRANSIENT_GROUP: + * + * The name of a `GKeyFile` group for data that should not + * be carried across upgrades. For more information, + * see ostree_deployment_origin_remove_transient_state(). + * + * Since: 2018.3 + */ +#define OSTREE_ORIGIN_TRANSIENT_GROUP "libostree-transient" + typedef struct _OstreeDeployment OstreeDeployment; _OSTREE_PUBLIC @@ -62,6 +73,7 @@ OstreeBootconfigParser *ostree_deployment_get_bootconfig (OstreeDeployment *self _OSTREE_PUBLIC GKeyFile *ostree_deployment_get_origin (OstreeDeployment *self); + _OSTREE_PUBLIC void ostree_deployment_set_index (OstreeDeployment *self, int index); _OSTREE_PUBLIC @@ -71,6 +83,9 @@ void ostree_deployment_set_bootconfig (OstreeDeployment *self, OstreeBootconfigP _OSTREE_PUBLIC void ostree_deployment_set_origin (OstreeDeployment *self, GKeyFile *origin); +_OSTREE_PUBLIC +void ostree_deployment_origin_remove_transient_state (GKeyFile *origin); + _OSTREE_PUBLIC OstreeDeployment *ostree_deployment_clone (OstreeDeployment *self); diff --git a/src/ostree/ot-admin-builtin-upgrade.c b/src/ostree/ot-admin-builtin-upgrade.c index e42ded6c..1b6a0253 100644 --- a/src/ostree/ot-admin-builtin-upgrade.c +++ b/src/ostree/ot-admin-builtin-upgrade.c @@ -88,33 +88,18 @@ ot_admin_builtin_upgrade (int argc, char **argv, OstreeCommandInvocation *invoca g_autoptr(GKeyFile) origin = ostree_sysroot_upgrader_dup_origin (upgrader); if (origin != NULL) { - gboolean origin_changed = FALSE; - + /* Should we consider requiring --discard-hotfix here? */ + ostree_deployment_origin_remove_transient_state (origin); if (opt_override_commit != NULL) { /* Override the commit to pull and deploy. */ g_key_file_set_string (origin, "origin", "override-commit", opt_override_commit); - origin_changed = TRUE; - } - else - { - /* Strip any override-commit from the origin file so - * we always upgrade to the latest available commit. */ - origin_changed = g_key_file_remove_key (origin, "origin", - "override-commit", NULL); } - /* Should we consider requiring --discard-hotfix here? */ - origin_changed |= g_key_file_remove_key (origin, "origin", "unlocked", NULL); - - if (origin_changed) - { - /* XXX GCancellable parameter is not used. */ - if (!ostree_sysroot_upgrader_set_origin (upgrader, origin, NULL, error)) - return FALSE; - } + if (!ostree_sysroot_upgrader_set_origin (upgrader, origin, NULL, error)) + return FALSE; } gboolean changed; From 7f88fddcd41f0fb12333eba145c99d2499b7767f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 23 Feb 2018 12:46:32 -0500 Subject: [PATCH 09/51] =?UTF-8?q?sysroot:=20Add=20concept=20of=20deploymen?= =?UTF-8?q?t=20"pinning"=20=F0=9F=93=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Example user story: Jane rebases her OS to a new major version N, and wants to keep around N-1 even after a few upgrades for a while so she can easily roll back. I plan to add `rpm-ostree rebase --pin` to opt-in to this for example. Builds on the new `libostree-transient` group to store pinning state there. Closes: https://github.com/ostreedev/ostree/issues/1460 Closes: #1464 Approved by: jlebon --- Makefile-man.am | 1 + Makefile-ostree.am | 1 + apidoc/ostree-sections.txt | 2 + bash/ostree | 31 +++++++++++ man/ostree-admin-pin.xml | 82 ++++++++++++++++++++++++++++ src/libostree/libostree-devel.sym | 2 + src/libostree/ostree-deployment.c | 17 ++++++ src/libostree/ostree-deployment.h | 3 + src/libostree/ostree-sysroot.c | 44 +++++++++++++++ src/libostree/ostree-sysroot.h | 6 ++ src/ostree/ot-admin-builtin-pin.c | 80 +++++++++++++++++++++++++++ src/ostree/ot-admin-builtin-status.c | 2 + src/ostree/ot-admin-builtins.h | 1 + src/ostree/ot-builtin-admin.c | 3 + tests/test-admin-deploy-2.sh | 49 ++++++++++++++++- 15 files changed, 323 insertions(+), 1 deletion(-) create mode 100644 man/ostree-admin-pin.xml create mode 100644 src/ostree/ot-admin-builtin-pin.c diff --git a/Makefile-man.am b/Makefile-man.am index e2f88a16..4d99cde1 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -26,6 +26,7 @@ ostree-admin-config-diff.1 ostree-admin-deploy.1 \ ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 \ ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 \ ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin-unlock.1 \ +ostree-admin-pin.1 \ ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 \ ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 \ ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 \ diff --git a/Makefile-ostree.am b/Makefile-ostree.am index c366c84f..cccbe300 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -74,6 +74,7 @@ ostree_SOURCES += \ src/ostree/ot-admin-builtin-set-origin.c \ src/ostree/ot-admin-builtin-status.c \ src/ostree/ot-admin-builtin-switch.c \ + src/ostree/ot-admin-builtin-pin.c \ src/ostree/ot-admin-builtin-upgrade.c \ src/ostree/ot-admin-builtin-unlock.c \ src/ostree/ot-admin-builtins.h \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 892aa0c0..55f2e7a9 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -169,6 +169,7 @@ ostree_deployment_get_bootconfig ostree_deployment_get_origin ostree_deployment_get_origin_relpath ostree_deployment_get_unlocked +ostree_deployment_is_pinned ostree_deployment_set_index ostree_deployment_set_bootserial ostree_deployment_set_bootconfig @@ -509,6 +510,7 @@ ostree_sysroot_init_osname ostree_sysroot_deployment_set_kargs ostree_sysroot_deployment_set_mutable ostree_sysroot_deployment_unlock +ostree_sysroot_deployment_set_pinned ostree_sysroot_write_deployments ostree_sysroot_write_deployments_with_options ostree_sysroot_write_origin_file diff --git a/bash/ostree b/bash/ostree index fe8e3d2c..edef6cff 100644 --- a/bash/ostree +++ b/bash/ostree @@ -399,6 +399,37 @@ _ostree_admin_os_init() { return 0 } +_ostree_admin_pin() { + local boolean_options=" + $main_boolean_options + " + + local options_with_args=" + --sysroot + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --sysroot) + __ostree_compreply_dirs_only + return 0 + ;; + $options_with_args_glob ) + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + esac + + return 0 +} + _ostree_admin_set_origin() { local boolean_options=" $main_boolean_options diff --git a/man/ostree-admin-pin.xml b/man/ostree-admin-pin.xml new file mode 100644 index 00000000..db0787ae --- /dev/null +++ b/man/ostree-admin-pin.xml @@ -0,0 +1,82 @@ + + + + + + + + + ostree admin pin + OSTree + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + ostree admin pin + 1 + + + + ostree-admin-pin + Explicitly retain deployment at a given index + + + + + ostree admin pin INDEX + + + + + Description + + + Ensures the deployment at INDEX, will not be garbage + collected by default. This is termed "pinning". If the + -u option is provided, undoes a pinning operation. + + + + + Options + + + + , + + + Undoes a pinning operation. + + + + + + diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 123627a4..285ba5f5 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,6 +20,8 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2018.3 { ostree_deployment_origin_remove_transient_state; + ostree_sysroot_deployment_set_pinned; + ostree_deployment_is_pinned; } LIBOSTREE_2018.2; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-deployment.c b/src/libostree/ostree-deployment.c index 2c479fdb..75a5bd1d 100644 --- a/src/libostree/ostree-deployment.c +++ b/src/libostree/ostree-deployment.c @@ -322,3 +322,20 @@ ostree_deployment_get_unlocked (OstreeDeployment *self) { return self->unlocked; } + +/** + * ostree_deployment_is_pinned: + * @self: Deployment + * + * See ostree_sysroot_deployment_set_pinned(). + * + * Returns: `TRUE` if deployment will not be subject to GC + * Since: 2018.3 + */ +gboolean +ostree_deployment_is_pinned (OstreeDeployment *self) +{ + if (!self->origin) + return FALSE; + return g_key_file_get_boolean (self->origin, OSTREE_ORIGIN_TRANSIENT_GROUP, "pinned", NULL); +} diff --git a/src/libostree/ostree-deployment.h b/src/libostree/ostree-deployment.h index 985c8133..612222a2 100644 --- a/src/libostree/ostree-deployment.h +++ b/src/libostree/ostree-deployment.h @@ -74,6 +74,9 @@ _OSTREE_PUBLIC GKeyFile *ostree_deployment_get_origin (OstreeDeployment *self); +_OSTREE_PUBLIC +gboolean ostree_deployment_is_pinned (OstreeDeployment *self); + _OSTREE_PUBLIC void ostree_deployment_set_index (OstreeDeployment *self, int index); _OSTREE_PUBLIC diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 34799444..2c12b78b 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1572,12 +1572,14 @@ ostree_sysroot_simple_write_deployment (OstreeSysroot *sysroot, /* Retain deployment if: * - we're explicitly asked to, or + * - it's pinned * - the deployment is for another osname, or * - we're keeping pending deployments and this is a pending deployment, or * - this is the merge or boot deployment, or * - we're keeping rollback deployments and this is a rollback deployment */ if (retain + || ostree_deployment_is_pinned (deployment) || !osname_matches || (retain_pending && !passed_crossover) || (is_booted || is_merge) @@ -1832,3 +1834,45 @@ ostree_sysroot_deployment_unlock (OstreeSysroot *self, return TRUE; } + +/** + * ostree_sysroot_deployment_set_pinned: + * @self: Sysroot + * @deployment: A deployment + * @is_pinned: Whether or not deployment will be automatically GC'd + * @error: Error + * + * By default, deployments may be subject to garbage collection. Typical uses of + * libostree only retain at most 2 deployments. If @is_pinned is `TRUE`, a + * metadata bit will be set causing libostree to avoid automatic GC of the + * deployment. However, this is really an "advisory" note; it's still possible + * for e.g. older versions of libostree unaware of pinning to GC the deployment. + * + * This function does nothing and returns successfully if the deployment + * is already in the desired pinning state. + * + * Since: 2018.3 + */ +gboolean +ostree_sysroot_deployment_set_pinned (OstreeSysroot *self, + OstreeDeployment *deployment, + gboolean is_pinned, + GError **error) +{ + const gboolean current_pin = ostree_deployment_is_pinned (deployment); + if (is_pinned == current_pin) + return TRUE; + + g_autoptr(OstreeDeployment) deployment_clone = ostree_deployment_clone (deployment); + GKeyFile *origin_clone = ostree_deployment_get_origin (deployment_clone); + + if (is_pinned) + g_key_file_set_boolean (origin_clone, OSTREE_ORIGIN_TRANSIENT_GROUP, "pinned", TRUE); + else + g_key_file_remove_key (origin_clone, OSTREE_ORIGIN_TRANSIENT_GROUP, "pinned", NULL); + + if (!ostree_sysroot_write_origin_file (self, deployment, origin_clone, NULL, error)) + return FALSE; + + return TRUE; +} diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index 830ed272..e4763d37 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -181,6 +181,12 @@ gboolean ostree_sysroot_deployment_set_mutable (OstreeSysroot *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sysroot_deployment_set_pinned (OstreeSysroot *self, + OstreeDeployment *deployment, + gboolean is_pinned, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sysroot_deployment_unlock (OstreeSysroot *self, OstreeDeployment *deployment, diff --git a/src/ostree/ot-admin-builtin-pin.c b/src/ostree/ot-admin-builtin-pin.c new file mode 100644 index 00000000..e26ea926 --- /dev/null +++ b/src/ostree/ot-admin-builtin-pin.c @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2018 Colin Walters + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include + +#include "ot-main.h" +#include "ot-admin-builtins.h" +#include "ot-admin-functions.h" +#include "ostree.h" +#include "otutil.h" + +/* ATTENTION: + * Please remember to update the bash-completion script (bash/ostree) and + * man page (man/ostree-admin-pin.xml) when changing the option list. + */ + +static gboolean opt_unpin; + +static GOptionEntry options[] = { + { "unpin", 'u', 0, G_OPTION_ARG_NONE, &opt_unpin, "Unset pin", NULL }, + { NULL } +}; + +gboolean +ot_admin_builtin_pin (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(GOptionContext) context = g_option_context_new ("INDEX"); + g_autoptr(OstreeSysroot) sysroot = NULL; + if (!ostree_admin_option_context_parse (context, options, &argc, &argv, + OSTREE_ADMIN_BUILTIN_FLAG_SUPERUSER, + invocation, &sysroot, cancellable, error)) + return FALSE; + + if (argc < 2) + { + ot_util_usage_error (context, "INDEX must be specified", error); + return FALSE; + } + + const char *deploy_index_str = argv[1]; + const int deploy_index = atoi (deploy_index_str); + + g_autoptr(OstreeDeployment) target_deployment = ot_admin_get_indexed_deployment (sysroot, deploy_index, error); + if (!target_deployment) + return FALSE; + + + gboolean current_pin = ostree_deployment_is_pinned (target_deployment); + const gboolean desired_pin = !opt_unpin; + if (current_pin == desired_pin) + g_print ("Deployment is already %s\n", current_pin ? "pinned" : "unpinned"); + else + { + if (!ostree_sysroot_deployment_set_pinned (sysroot, target_deployment, desired_pin, error)) + return FALSE; + g_print ("Deployment is now %s\n", desired_pin ? "pinned" : "unpinned"); + } + + return TRUE; +} diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 0279c5af..02991309 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -149,6 +149,8 @@ ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocat ostree_deployment_unlocked_state_to_string (unlocked), red_bold_suffix); } + if (ostree_deployment_is_pinned (deployment)) + g_print (" Pinned: yes\n"); if (!origin) g_print (" origin: none\n"); else diff --git a/src/ostree/ot-admin-builtins.h b/src/ostree/ot-admin-builtins.h index 39996243..a81f4d62 100644 --- a/src/ostree/ot-admin-builtins.h +++ b/src/ostree/ot-admin-builtins.h @@ -39,6 +39,7 @@ BUILTINPROTO(init_fs); BUILTINPROTO(undeploy); BUILTINPROTO(deploy); BUILTINPROTO(cleanup); +BUILTINPROTO(pin); BUILTINPROTO(unlock); BUILTINPROTO(status); BUILTINPROTO(set_origin); diff --git a/src/ostree/ot-builtin-admin.c b/src/ostree/ot-builtin-admin.c index f4e687e9..fd6d9a8f 100644 --- a/src/ostree/ot-builtin-admin.c +++ b/src/ostree/ot-builtin-admin.c @@ -54,6 +54,9 @@ static OstreeCommand admin_subcommands[] = { { "set-origin", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_set_origin, "Set Origin and create a new origin file" }, + { "pin", OSTREE_BUILTIN_FLAG_NO_REPO, + ot_admin_builtin_pin, + "Change the \"pinning\" state of a deployment" }, { "status", OSTREE_BUILTIN_FLAG_NO_REPO, ot_admin_builtin_status, "List deployments" }, diff --git a/tests/test-admin-deploy-2.sh b/tests/test-admin-deploy-2.sh index 4ecaf67a..eab0a3d3 100755 --- a/tests/test-admin-deploy-2.sh +++ b/tests/test-admin-deploy-2.sh @@ -26,7 +26,7 @@ set -euo pipefail # Exports OSTREE_SYSROOT so --sysroot not needed. setup_os_repository "archive" "syslinux" -echo "1..3" +echo "1..6" ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmaster/x86_64-runtime rev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) @@ -63,3 +63,50 @@ assert_has_dir sysroot/boot/ostree/testos-${bootcsum} assert_file_has_content sysroot/ostree/deploy/testos/deploy/${newrev}.0/etc/os-release 'NAME=TestOS' echo "ok manual cleanup" + +assert_n_pinned() { + local n=$1 + ${CMD_PREFIX} ostree admin status > status.txt + local n_pinned="$(grep -F -c -e 'Pinned: yes' < status.txt)" + if test "${n_pinned}" '!=' "${n}"; then + cat status.txt + fatal "${n_pinned} != ${n}" + fi +} +assert_n_deployments() { + local n=$1 + ${CMD_PREFIX} ostree admin status > status.txt + local n_deployments="$(grep -F -c -e 'Version: ' < status.txt)" + if test "${n_deployments}" '!=' "${n}"; then + cat status.txt + fatal "${n_deployments} != ${n}" + fi +} +assert_n_pinned 0 +${CMD_PREFIX} ostree admin pin 0 +assert_n_pinned 1 +${CMD_PREFIX} ostree admin pin -u 0 +assert_n_pinned 0 +echo "ok pin unpin" + +${CMD_PREFIX} ostree admin pin 0 +${CMD_PREFIX} ostree admin pin 1 +assert_n_pinned 2 +assert_n_deployments 2 +os_repository_new_commit +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_n_pinned 2 +assert_n_deployments 3 +echo "ok pin across upgrades" + +${CMD_PREFIX} ostree admin pin -u 1 +os_repository_new_commit +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_n_pinned 1 +assert_n_deployments 3 +os_repository_new_commit +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_n_pinned 1 +assert_n_deployments 3 + +echo "ok pinning" From ee1f6b23153fea4324b9bdf6de417863931b6aa1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 26 Feb 2018 14:11:00 -0500 Subject: [PATCH 10/51] bash-completion: Remove `admin` completions The `admin` commandline should be considered a demo; I just added the `pin` command *mostly* so we could use it for unit tests, although I can imagine other people using it. But maintaining completions is a lot of overhead right now, let's not do it for `admin`. The other command line options that operate on repos we will definitely maintain since they're used in releng contexts. Closes: #1468 Approved by: jlebon --- bash/ostree | 497 ------------------ src/ostree/ot-admin-builtin-cleanup.c | 5 - src/ostree/ot-admin-builtin-deploy.c | 5 - src/ostree/ot-admin-builtin-diff.c | 5 - src/ostree/ot-admin-builtin-init-fs.c | 5 - src/ostree/ot-admin-builtin-os-init.c | 5 - src/ostree/ot-admin-builtin-pin.c | 5 - src/ostree/ot-admin-builtin-set-origin.c | 5 - src/ostree/ot-admin-builtin-status.c | 5 - src/ostree/ot-admin-builtin-switch.c | 5 - src/ostree/ot-admin-builtin-undeploy.c | 5 - src/ostree/ot-admin-builtin-unlock.c | 5 - src/ostree/ot-admin-builtin-upgrade.c | 5 - ...ot-admin-instutil-builtin-grub2-generate.c | 5 - ...-instutil-builtin-selinux-ensure-labeled.c | 5 - .../ot-admin-instutil-builtin-set-kargs.c | 5 - 16 files changed, 572 deletions(-) diff --git a/bash/ostree b/bash/ostree index edef6cff..218e4254 100644 --- a/bash/ostree +++ b/bash/ostree @@ -180,503 +180,6 @@ _ostree_ostree() { return 0 } -_ostree_admin_cleanup() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_config_diff() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --os - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --os) - __ostree_compreply_oses - return 0 - ;; - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_deploy() { - local boolean_options=" - $main_boolean_options - --retain - --retain-pending - --retain-rollback - --not-as-default - --karg-proc-cmdline - " - - local options_with_args=" - --karg - --karg-append - --origin-file - --os - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --origin-file) - __ostree_compreply_all_files - return 0 - ;; - --os) - __ostree_compreply_oses - return 0 - ;; - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_init_fs() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_instutil() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - *) - local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) - if [ $cword -eq $argpos ]; then - local instutil_commands=" - grub2-generate - selinux-ensure-labeled - set-kargs - " - COMPREPLY=( $( compgen -W "$instutil_commands" -- "$cur" ) ) - fi - ;; - esac - - return 0 -} - -_ostree_admin_os_init() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_pin() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_set_origin() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --index - --set -s - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - *) - local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) - if [ $cword -eq $argpos ]; then - __ostree_compreply_remotes - fi - ;; - esac - - return 0 -} - -_ostree_admin_status() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_switch() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --os - --reboot -r - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --os) - __ostree_compreply_oses - return 0 - ;; - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - *) - local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) - - if [ $cword -eq $argpos ]; then - __ostree_compreply_refs - fi - esac - - return 0 -} - -_ostree_admin_undeploy() { - local boolean_options=" - $main_boolean_options - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_unlock() { - local boolean_options=" - $main_boolean_options - --hotfix - " - - local options_with_args=" - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin_upgrade() { - local boolean_options=" - $main_boolean_options - --allow-downgrade - --deploy-only - --pull-only - --reboot -r - " - - local options_with_args=" - --os - --override-commit - --sysroot - " - - local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) - - case "$prev" in - --override-commit) - __ostree_compreply_commits - return 0 - ;; - --sysroot) - __ostree_compreply_dirs_only - return 0 - ;; - $options_with_args_glob ) - return 0 - ;; - esac - - case "$cur" in - -*) - local all_options="$boolean_options $options_with_args" - __ostree_compreply_all_options - ;; - esac - - return 0 -} - -_ostree_admin() { - local subcommands=" - cleanup - config-diff - deploy - init-fs - instutil - os-init - set-origin - status - switch - undeploy - unlock - upgrade - " - - __ostree_subcommands "$subcommands" && return 0 - - case "$cur" in - -*) - COMPREPLY=( $( compgen -W "$main_boolean_options" -- "$cur" ) ) - ;; - *) - COMPREPLY=( $( compgen -W "$subcommands" -- "$cur" ) ) - ;; - esac - - return 0 -} - _ostree_cat() { local boolean_options=" $main_boolean_options diff --git a/src/ostree/ot-admin-builtin-cleanup.c b/src/ostree/ot-admin-builtin-cleanup.c index 4028931e..a4753030 100644 --- a/src/ostree/ot-admin-builtin-cleanup.c +++ b/src/ostree/ot-admin-builtin-cleanup.c @@ -30,11 +30,6 @@ #include -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-cleanup.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index ee7f3ccf..d9905212 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -45,11 +45,6 @@ static char *opt_osname; static char *opt_origin_path; static gboolean opt_kernel_arg_none; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-deploy.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, { "origin-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_origin_path, "Specify origin file", "FILENAME" }, diff --git a/src/ostree/ot-admin-builtin-diff.c b/src/ostree/ot-admin-builtin-diff.c index 6a44a72b..27855881 100644 --- a/src/ostree/ot-admin-builtin-diff.c +++ b/src/ostree/ot-admin-builtin-diff.c @@ -32,11 +32,6 @@ static char *opt_osname; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-config-diff.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, { NULL } diff --git a/src/ostree/ot-admin-builtin-init-fs.c b/src/ostree/ot-admin-builtin-init-fs.c index 70348402..cca63a62 100644 --- a/src/ostree/ot-admin-builtin-init-fs.c +++ b/src/ostree/ot-admin-builtin-init-fs.c @@ -30,11 +30,6 @@ #include -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-init-fs.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-builtin-os-init.c b/src/ostree/ot-admin-builtin-os-init.c index 411e415e..203f297b 100644 --- a/src/ostree/ot-admin-builtin-os-init.c +++ b/src/ostree/ot-admin-builtin-os-init.c @@ -30,11 +30,6 @@ #include -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-os-init.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-builtin-pin.c b/src/ostree/ot-admin-builtin-pin.c index e26ea926..f110983b 100644 --- a/src/ostree/ot-admin-builtin-pin.c +++ b/src/ostree/ot-admin-builtin-pin.c @@ -29,11 +29,6 @@ #include "ostree.h" #include "otutil.h" -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-pin.xml) when changing the option list. - */ - static gboolean opt_unpin; static GOptionEntry options[] = { diff --git a/src/ostree/ot-admin-builtin-set-origin.c b/src/ostree/ot-admin-builtin-set-origin.c index 77f14bf5..07453a87 100644 --- a/src/ostree/ot-admin-builtin-set-origin.c +++ b/src/ostree/ot-admin-builtin-set-origin.c @@ -35,11 +35,6 @@ static int opt_index = -1; static char **opt_set; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-set-origin.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "set", 's', 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, { "index", 0, 0, G_OPTION_ARG_INT, &opt_index, "Operate on the deployment INDEX, starting from zero", "INDEX" }, diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index 02991309..a2e24ef3 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -30,11 +30,6 @@ #include -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-status.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-builtin-switch.c b/src/ostree/ot-admin-builtin-switch.c index 7609b25a..2f12ef1d 100644 --- a/src/ostree/ot-admin-builtin-switch.c +++ b/src/ostree/ot-admin-builtin-switch.c @@ -34,11 +34,6 @@ static gboolean opt_reboot; static char *opt_osname; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-switch.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "reboot", 'r', 0, G_OPTION_ARG_NONE, &opt_reboot, "Reboot after switching trees", NULL }, { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, diff --git a/src/ostree/ot-admin-builtin-undeploy.c b/src/ostree/ot-admin-builtin-undeploy.c index 6f070524..0a50dc0d 100644 --- a/src/ostree/ot-admin-builtin-undeploy.c +++ b/src/ostree/ot-admin-builtin-undeploy.c @@ -29,11 +29,6 @@ #include "ostree.h" #include "otutil.h" -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-undeploy.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-builtin-unlock.c b/src/ostree/ot-admin-builtin-unlock.c index 34ff08cd..f0efa44a 100644 --- a/src/ostree/ot-admin-builtin-unlock.c +++ b/src/ostree/ot-admin-builtin-unlock.c @@ -34,11 +34,6 @@ static gboolean opt_hotfix; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-unlock.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "hotfix", 0, 0, G_OPTION_ARG_NONE, &opt_hotfix, "Retain changes across reboots", NULL }, { NULL } diff --git a/src/ostree/ot-admin-builtin-upgrade.c b/src/ostree/ot-admin-builtin-upgrade.c index 1b6a0253..eafe8bce 100644 --- a/src/ostree/ot-admin-builtin-upgrade.c +++ b/src/ostree/ot-admin-builtin-upgrade.c @@ -40,11 +40,6 @@ static gboolean opt_deploy_only; static char *opt_osname; static char *opt_override_commit; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-upgrade.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, { "reboot", 'r', 0, G_OPTION_ARG_NONE, &opt_reboot, "Reboot after a successful upgrade", NULL }, diff --git a/src/ostree/ot-admin-instutil-builtin-grub2-generate.c b/src/ostree/ot-admin-instutil-builtin-grub2-generate.c index df9d804b..3ab5c245 100644 --- a/src/ostree/ot-admin-instutil-builtin-grub2-generate.c +++ b/src/ostree/ot-admin-instutil-builtin-grub2-generate.c @@ -28,11 +28,6 @@ #include "otutil.h" -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-instutil.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c b/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c index 3b546711..8bf75b31 100644 --- a/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c +++ b/src/ostree/ot-admin-instutil-builtin-selinux-ensure-labeled.c @@ -173,11 +173,6 @@ selinux_relabel_dir (OstreeSePolicy *sepolicy, return ret; } -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-instutil.xml) when changing the option list. - */ - static GOptionEntry options[] = { { NULL } }; diff --git a/src/ostree/ot-admin-instutil-builtin-set-kargs.c b/src/ostree/ot-admin-instutil-builtin-set-kargs.c index 1194f82d..666e5369 100644 --- a/src/ostree/ot-admin-instutil-builtin-set-kargs.c +++ b/src/ostree/ot-admin-instutil-builtin-set-kargs.c @@ -34,11 +34,6 @@ static gboolean opt_merge; static char **opt_replace; static char **opt_append; -/* ATTENTION: - * Please remember to update the bash-completion script (bash/ostree) and - * man page (man/ostree-admin-instutil.xml) when changing the option list. - */ - static GOptionEntry options[] = { { "import-proc-cmdline", 0, 0, G_OPTION_ARG_NONE, &opt_proc_cmdline, "Import current /proc/cmdline", NULL }, { "merge", 0, 0, G_OPTION_ARG_NONE, &opt_merge, "Merge with previous command line", NULL }, From 6db6268dfd76dfb36bd7bd756fe78ba8a1abcd5b Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Mon, 26 Feb 2018 18:54:54 -0800 Subject: [PATCH 11/51] lib/repo-finder-mount: Fix path to flatpak repo OstreeRepoFinderMount checks mounts for a few well-known directories such as "ostree/repo" and ".ostree/repo" to try to find remotes. One of the hard-coded directories is "var/lib/flatpak" but that's the flatpak directory, not the ostree repo used by flatpak, which is at "var/lib/flatpak/repo". So this commit changes the path so the repo can be found. For recent versions of Endless, flatpak uses /ostree/repo as its repository, so this commit won't make a difference there. But it may help on other operating systems. Closes: #1471 Approved by: cgwalters --- src/libostree/ostree-repo-finder-mount.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index a7919fca..cdb958cd 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -439,7 +439,7 @@ G_GNUC_END_IGNORE_DEPRECATIONS { ".ostree/repo", "ostree/repo", - "var/lib/flatpak", + "var/lib/flatpak/repo", }; for (i = 0; i < G_N_ELEMENTS (well_known_repos); i++) From 530043fcf6444f5751caa2c478d9411ab78e1698 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 28 Feb 2018 17:30:18 +0000 Subject: [PATCH 12/51] lib/sysroot: Fix retrieving non-booted pending deployment If we're booted into a deployment, then any queries for the pending merge deployment of a non-booted OS will fail due all of them being considered rollback. Fix this by filtering by `osname` *before* determining if we've crossed the booted deployment yet. Closes: #1472 Approved by: cgwalters --- src/libostree/ostree-sysroot.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 2c12b78b..2d12deb6 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1202,6 +1202,10 @@ ostree_sysroot_query_deployments_for (OstreeSysroot *self, { OstreeDeployment *deployment = self->deployments->pdata[i]; + /* Ignore deployments not for this osname */ + if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) + continue; + /* Is this deployment booted? If so, note we're past the booted */ if (self->booted_deployment != NULL && ostree_deployment_equal (deployment, self->booted_deployment)) @@ -1210,10 +1214,6 @@ ostree_sysroot_query_deployments_for (OstreeSysroot *self, continue; } - /* Ignore deployments not for this osname */ - if (strcmp (ostree_deployment_get_osname (deployment), osname) != 0) - continue; - if (!found_booted && !ret_pending) ret_pending = g_object_ref (deployment); else if (found_booted && !ret_rollback) From 2381ca0aa4db8d8cc9ab81acd761a9ee96279352 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Feb 2018 16:06:31 -0800 Subject: [PATCH 13/51] lib/repo-pull: Fix free function for hash table The "ref_original_commits" hash table uses string values, not variants, so fix the free function passed to g_hash_table_new_full (). Since g_variant_unref isn't NULL safe, this prevents an assertion failure when a NULL value is inserted. Dan Nicholson suggested this patch; I'm just submitting it because he's busy. Fixes https://github.com/ostreedev/ostree/issues/1433 Closes: #1474 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 999d7ecf..75de5bdd 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3365,7 +3365,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_free); pull_data->ref_original_commits = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal, (GDestroyNotify)NULL, - (GDestroyNotify)g_variant_unref); + (GDestroyNotify)g_free); pull_data->gpg_verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, From d0e4a4f03bb1288b9dbb1e8643de35166e348db0 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 28 Feb 2018 15:58:41 -0800 Subject: [PATCH 14/51] lib/repo-finder-mount: Update comment about paths This updates the gtk-doc comment for OstreeRepoFinderMount to match the correct flatpak repo path, which was fixed in commit 6db6268df. Closes: #1473 Approved by: cgwalters --- src/libostree/ostree-repo-finder-mount.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index cdb958cd..41aefb75 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -52,7 +52,7 @@ * enumerated, and all OSTree repositories below it will be searched, in lexical * order, for the requested #OstreeCollectionRefs. The names of the directories * below `.ostree/repos.d` are irrelevant, apart from their lexical ordering. - * The directories `.ostree/repo`, `ostree/repo` and `var/lib/flatpak` + * The directories `.ostree/repo`, `ostree/repo` and `var/lib/flatpak/repo` * will be searched after the others, if they exist. * Non-removable volumes are ignored. * From 7727fdd9f7833cb4f7181a71e1f84f3aa2e183e4 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 1 Mar 2018 15:44:42 -0800 Subject: [PATCH 15/51] lib/remote: Fix memory leak Closes: #1476 Approved by: cgwalters --- src/libostree/ostree-remote.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-remote.c b/src/libostree/ostree-remote.c index da513ab4..da325d18 100644 --- a/src/libostree/ostree-remote.c +++ b/src/libostree/ostree-remote.c @@ -149,6 +149,7 @@ ostree_remote_unref (OstreeRemote *remote) if (g_atomic_int_dec_and_test (&remote->ref_count)) { g_clear_pointer (&remote->name, g_free); + g_clear_pointer (&remote->refspec_name, g_free); g_clear_pointer (&remote->group, g_free); g_clear_pointer (&remote->keyring, g_free); g_clear_object (&remote->file); From 1214395f0e08c9e895e0983edaee4648b07ec69c Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 1 Mar 2018 15:43:38 -0800 Subject: [PATCH 16/51] lib/repo-finder-mount: Improve debug message This makes it easier to tell which mount is being checked when repos are found. Closes: #1477 Approved by: cgwalters --- src/libostree/ostree-repo-finder-mount.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c index 41aefb75..41a6bed2 100644 --- a/src/libostree/ostree-repo-finder-mount.c +++ b/src/libostree/ostree-repo-finder-mount.c @@ -295,8 +295,8 @@ scan_and_add_repo (int dfd, }; g_array_append_val (inout_repos_refs, val); - g_debug ("%s: Adding repo ‘%s’ (%ssortable)", - G_STRFUNC, path, sortable ? "" : "not "); + g_debug ("%s: Adding repo ‘%s’ on mount ‘%s’ (%ssortable)", + G_STRFUNC, path, mount_name, sortable ? "" : "not "); } } From 733c0498dca8926e9078563240bd06a488fcd527 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Mar 2018 10:56:45 -0500 Subject: [PATCH 17/51] lib/repo: Do account for size with prune --no-prune I think this got changed in a refactor. We definitely want to total up the amount of space that *would* be freed even with `--no-prune` AKA `OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE`. It's actually a bit terrifying this is apparently the first test case for the `--no-prune` option... Closes: https://github.com/ostreedev/ostree/issues/1480 Closes: #1483 Approved by: jlebon --- src/libostree/ostree-repo-prune.c | 17 ++++++++++------- tests/test-prune.sh | 7 +++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index fc3cfa54..c8a178ec 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -52,28 +52,31 @@ maybe_prune_loose_object (OtPruneData *data, if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) { + guint64 storage_size = 0; + g_debug ("Pruning unneeded object %s.%s", checksum, ostree_object_type_to_string (objtype)); + + if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, + &storage_size, cancellable, error)) + return FALSE; + + data->freed_bytes += storage_size; + if (!(flags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)) { - guint64 storage_size = 0; - if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_repo_mark_commit_partial (data->repo, checksum, FALSE, error)) return FALSE; } - if (!ostree_repo_query_object_storage_size (data->repo, objtype, checksum, - &storage_size, cancellable, error)) - return FALSE; - if (!ostree_repo_delete_object (data->repo, objtype, checksum, cancellable, error)) return FALSE; - data->freed_bytes += storage_size; } + if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_unreachable_meta++; else diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 0f083221..7de372c8 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -52,6 +52,13 @@ assert_repo_has_n_commits() { assert_streq "$(find ${repo}/objects -name '*.commit' | wc -l)" "${count}" } +# Test --no-prune +objectcount_orig=$(find repo/objects | wc -l) +${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 --no-prune | tee noprune.txt +assert_file_has_content noprune.txt 'Would delete: [1-9][0-9]* objects, freeing [1-9][0-9]*' +objectcount_new=$(find repo/objects | wc -l) +assert_streq "${objectcount_orig}" "${objectcount_new}" + ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=2 -v assert_repo_has_n_commits repo 3 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount From 79d6f635df47e6519d0211d08cce89b1034562bd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Mar 2018 10:42:19 -0500 Subject: [PATCH 18/51] prune: Error if --static-deltas-only without --delete-commit The original changes here apparently had the *idea* that `--static-deltas-only` would be useful in general, but we never implemented that. The current situation where it's ignored unless `--delete-commit` is specified is very misleading and I can easily see it leading to data loss for people. Let's error out until we have a chance to make it actually useful. Related: https://github.com/ostreedev/ostree/issues/1479 Closes: #1482 Approved by: giuseppe --- src/ostree/ot-builtin-prune.c | 9 +++++++++ tests/test-prune.sh | 7 ++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 3f39dfe8..c34bbf4c 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -174,6 +174,15 @@ ostree_builtin_prune (int argc, char **argv, OstreeCommandInvocation *invocation else if (!delete_commit (repo, opt_delete_commit, cancellable, error)) return FALSE; } + else + { + /* In the future we should make this useful, but for now let's + * error out since what we were doing before was very misleading. + * https://github.com/ostreedev/ostree/issues/1479 + */ + if (opt_static_deltas_only) + return glnx_throw (error, "--static-deltas-only requires --delete-commit; see https://github.com/ostreedev/ostree/issues/1479"); + } OstreeRepoPruneFlags pruneflags = 0; if (opt_refs_only) diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 7de372c8..e8734801 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -141,9 +141,10 @@ assert_file_has_content deltascount "^1$" ${CMD_PREFIX} ostree --repo=repo static-delta generate test ${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount assert_file_has_content deltascount "^2$" -${CMD_PREFIX} ostree --repo=repo prune --static-deltas-only --keep-younger-than="October 20 2015" -${CMD_PREFIX} ostree --repo=repo static-delta list | wc -l > deltascount -assert_file_has_content deltascount "^1$" +if ${CMD_PREFIX} ostree --repo=repo prune --static-deltas-only --keep-younger-than="October 20 2015" 2>err.txt; then + fatal "pruned deltas only" +fi +assert_file_has_content_literal err.txt "--static-deltas-only requires --delete-commit" echo "ok prune" From fe6ae92ebc2940765008e4775d56be39661f3514 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Sat, 3 Mar 2018 12:57:12 -0800 Subject: [PATCH 19/51] lib: Fix memory leaks of OstreeRemote The _ostree_repo_get_remote() and _ostree_repo_get_remote_inherited() methods transfer ownership of the returned OstreeRemote to the caller, so this commit fixes a few call sites that weren't properly freeing it. Closes: #1478 Approved by: cgwalters --- src/libostree/ostree-repo-finder-config.c | 3 ++- src/libostree/ostree-repo-pull.c | 5 +++-- src/libostree/ostree-repo.c | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c index d6c8adda..76acb58e 100644 --- a/src/libostree/ostree-repo-finder-config.c +++ b/src/libostree/ostree-repo-finder-config.c @@ -30,6 +30,7 @@ #include #include +#include "ostree-autocleanups.h" #include "ostree-remote-private.h" #include "ostree-repo.h" #include "ostree-repo-private.h" @@ -176,7 +177,7 @@ ostree_repo_finder_config_resolve_async (OstreeRepoFinder *find while (g_hash_table_iter_next (&iter, (gpointer *) &remote_name, (gpointer *) &supported_ref_to_checksum)) { g_autoptr(GError) local_error = NULL; - OstreeRemote *remote; + g_autoptr(OstreeRemote) remote = NULL; /* We don’t know what last-modified timestamp the remote has without * making expensive HTTP queries, so leave that information blank. We diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 75de5bdd..766098cb 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -5645,7 +5645,7 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, { gsize i; g_auto(GStrv) remotes = NULL; - OstreeRemote *keyring_remote = NULL; + g_autoptr(OstreeRemote) keyring_remote = NULL; g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); g_return_val_if_fail (ostree_validate_collection_id (collection_id, NULL), NULL); @@ -5680,6 +5680,7 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, { g_debug ("%s: Ignoring remote ‘%s’ as it has no keyring configured.", G_STRFUNC, remotes[i]); + g_clear_object (&keyring_remote); continue; } @@ -5695,7 +5696,7 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, } if (keyring_remote != NULL) - return ostree_remote_ref (keyring_remote); + return g_steal_pointer (&keyring_remote); else { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 9242990c..20a8839b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4914,7 +4914,7 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, g_autofree char *gpgkeypath = NULL; /* Add the remote's keyring file if it exists. */ - OstreeRemote *remote; + g_autoptr(OstreeRemote) remote = NULL; remote = _ostree_repo_get_remote_inherited (self, remote_name, error); if (remote == NULL) @@ -4936,8 +4936,6 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, if (gpgkeypath) _ostree_gpg_verifier_add_key_ascii_file (verifier, gpgkeypath); - - ostree_remote_unref (remote); } if (add_global_keyring_dir) From 969e4eb72e9aff95024bc460b2d6b50bce5136f6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Mar 2018 17:31:12 -0500 Subject: [PATCH 20/51] repo/refs: Clean up error prefixing Add some "function global" prefixing in line with what we do in other places now, and drop the "manual filename" prefixing that is no longer necessary since https://github.com/GNOME/libglnx/commit/23f7df15006f14ddc3bc2ddee690f7f8604c3ebe Closes: https://github.com/ostreedev/ostree/issues/1467 Closes: #1485 Approved by: jlebon --- src/libostree/ostree-repo-refs.c | 19 +++++++------------ tests/basic-test.sh | 13 ++++++++++++- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index bb0e1c75..b96cfad0 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -617,6 +617,8 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Listing refs", error); + g_autofree char *remote = NULL; g_autofree char *ref_prefix = NULL; @@ -1029,10 +1031,7 @@ _ostree_repo_write_ref (OstreeRepo *self, { if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &dfd, error)) - { - g_prefix_error (error, "Opening %s: ", "refs/heads"); - return FALSE; - } + return FALSE; } else if (remote == NULL && ref->collection_id != NULL) { @@ -1041,10 +1040,7 @@ _ostree_repo_write_ref (OstreeRepo *self, /* refs/mirrors might not exist in older repositories, so create it. */ if (!glnx_shutil_mkdir_p_at_open (self->repo_dir_fd, "refs/mirrors", 0777, &refs_mirrors_dfd, cancellable, error)) - { - g_prefix_error (error, "Opening %s: ", "refs/mirrors"); - return FALSE; - } + return FALSE; if (rev != NULL) { @@ -1063,10 +1059,7 @@ _ostree_repo_write_ref (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, "refs/remotes", TRUE, &refs_remotes_dfd, error)) - { - g_prefix_error (error, "Opening %s: ", "refs/remotes"); - return FALSE; - } + return FALSE; if (rev != NULL) { @@ -1207,6 +1200,8 @@ ostree_repo_list_collection_refs (OstreeRepo *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Listing refs", error); + g_return_val_if_fail (OSTREE_IS_REPO (self), FALSE); g_return_val_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable), FALSE); g_return_val_if_fail (error == NULL || *error == NULL, FALSE); diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 3376ac58..b8e7eb07 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((82 + ${extra_basic_tests:-0}))" +echo "1..$((83 + ${extra_basic_tests:-0}))" CHECKOUT_U_ARG="" CHECKOUT_H_ARGS="-H" @@ -476,6 +476,17 @@ cd ${test_tmpdir} $OSTREE prune echo "ok prune didn't fail" +# https://github.com/ostreedev/ostree/issues/1467 +cd ${test_tmpdir} +mv repo/refs/remotes{,.orig} +if $OSTREE refs --list >/dev/null 2>err.txt; then + fatal "listed refs without remotes dir?" +fi +assert_file_has_content err.txt 'Listing refs.*opendir.*No such file or directory' +mv repo/refs/remotes{.orig,} +$OSTREE refs --list >/dev/null +echo "ok refs enoent error" + cd ${test_tmpdir} # Verify we can't cat dirs for path in / /baz; do From 1c9baad8a886399ef2362420fe34c4718d06f0de Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 5 Mar 2018 17:18:54 -0500 Subject: [PATCH 21/51] docs/prune: Document that --static-deltas-only isn't that useful This is the documentation followup to: https://github.com/ostreedev/ostree/pull/1482 See also https://github.com/ostreedev/ostree/issues/1481 Closes: #1484 Approved by: jlebon --- man/ostree-prune.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/man/ostree-prune.xml b/man/ostree-prune.xml index 641176ba..f1b517f3 100644 --- a/man/ostree-prune.xml +++ b/man/ostree-prune.xml @@ -114,8 +114,12 @@ Boston, MA 02111-1307, USA. =DEPTH - Change the behaviour of --keep-younger-than and --delete-commit to prune only - the static deltas files. + This option may currently only be used in combination with + . Previous versions of ostree silently accepted + the option without that, and ignored it. However, there are desired use + cases for pruning just static deltas (while retaining the commits), and it's + likely at some point this option will be supported for use cases outside of just + . From 418e4545de1855b9504a996877b0810a8cf5b6c6 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 14 Feb 2018 13:24:43 +0100 Subject: [PATCH 22/51] ostree-repo-private: remove declaration for _ostree_repo_find_object it was removed with: commit 8609cb036b935ce942214e9fdee6d90de0a210af Author: Colin Walters Date: Thu Apr 21 15:14:51 2016 -0400 repo: Simplify internal has_object() lookup code Signed-off-by: Giuseppe Scrivano Closes: #1443 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a31d4e5e..d61db28f 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -237,14 +237,6 @@ _ostree_repo_ensure_loose_objdir_at (int dfd, GCancellable *cancellable, GError **error); -gboolean -_ostree_repo_find_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *checksum, - GFile **out_stored_path, - GCancellable *cancellable, - GError **error); - GFile * _ostree_repo_get_commit_metadata_loose_path (OstreeRepo *self, const char *checksum); From 118f1f7e40004b079beea0a29f92cd51dc32f81f Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 2 Feb 2018 13:58:40 +0100 Subject: [PATCH 23/51] ostree: introduce PAYLOAD_LINK object type It will be used by successive commits to keep track of the payload checksum for objects stored in the repository. The goal is that files having the same payload but different xattrs can take advantage of reflinks where supported. Signed-off-by: Giuseppe Scrivano Closes: #1443 Approved by: cgwalters --- src/libostree/ostree-core.c | 5 +++++ src/libostree/ostree-core.h | 4 +++- src/libostree/ostree-repo.c | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index ba790dc7..33d6a48b 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1239,6 +1239,8 @@ ostree_object_type_to_string (OstreeObjectType objtype) return "tombstone-commit"; case OSTREE_OBJECT_TYPE_COMMIT_META: return "commitmeta"; + case OSTREE_OBJECT_TYPE_PAYLOAD_LINK: + return "payload-link"; default: g_assert_not_reached (); return NULL; @@ -1266,6 +1268,8 @@ ostree_object_type_from_string (const char *str) return OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT; else if (!strcmp (str, "commitmeta")) return OSTREE_OBJECT_TYPE_COMMIT_META; + else if (!strcmp (str, "payload-link")) + return OSTREE_OBJECT_TYPE_PAYLOAD_LINK; g_assert_not_reached (); return 0; } @@ -2122,6 +2126,7 @@ _ostree_validate_structureof_metadata (OstreeObjectType objtype, break; case OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT: case OSTREE_OBJECT_TYPE_COMMIT_META: + case OSTREE_OBJECT_TYPE_PAYLOAD_LINK: /* TODO */ break; case OSTREE_OBJECT_TYPE_FILE: diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 018f5070..b65c9ba9 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -68,6 +68,7 @@ G_BEGIN_DECLS * @OSTREE_OBJECT_TYPE_COMMIT: Toplevel object, refers to tree and dirmeta for root * @OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT: Toplevel object, refers to a deleted commit * @OSTREE_OBJECT_TYPE_COMMIT_META: Detached metadata for a commit + * @OSTREE_OBJECT_TYPE_PAYLOAD_LINK: Symlink to a .file given its checksum on the payload only. * * Enumeration for core object types; %OSTREE_OBJECT_TYPE_FILE is for * content, the other types are metadata. @@ -79,6 +80,7 @@ typedef enum { OSTREE_OBJECT_TYPE_COMMIT = 4, /* .commit */ OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT = 5, /* .commit-tombstone */ OSTREE_OBJECT_TYPE_COMMIT_META = 6, /* .commitmeta */ + OSTREE_OBJECT_TYPE_PAYLOAD_LINK = 7, /* .payload-link */ } OstreeObjectType; /** @@ -94,7 +96,7 @@ typedef enum { * * Last valid object type; use this to validate ranges. */ -#define OSTREE_OBJECT_TYPE_LAST OSTREE_OBJECT_TYPE_COMMIT_META +#define OSTREE_OBJECT_TYPE_LAST OSTREE_OBJECT_TYPE_PAYLOAD_LINK /** * OSTREE_DIRMETA_GVARIANT_FORMAT: diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 20a8839b..8d94f71a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3278,6 +3278,8 @@ list_loose_objects_at (OstreeRepo *self, objtype = OSTREE_OBJECT_TYPE_DIR_META; else if (strcmp (dot, ".commit") == 0) objtype = OSTREE_OBJECT_TYPE_COMMIT; + else if (strcmp (dot, ".payload-link") == 0) + objtype = OSTREE_OBJECT_TYPE_PAYLOAD_LINK; else continue; From 127d8bb846442c8b2767d12506d10d969b360663 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 2 Feb 2018 14:01:08 +0100 Subject: [PATCH 24/51] commit: add logic for .payload-link When a new object is added to the repository, create a $PAYLOAD-SHA256.payload-link symlink file as well. The target of the symlink is the checksum of the object that was added the repository. Whenever we add a new object file, in addition to lookup if the file is already present with the same checksum we also check if an object with the same payload is in the repository. If a file with the same payload is already present in the repository, we copy it with `glnx_regfile_copy_bytes` that internally attempts to create a reflink (ioctl (..., FICLONE, ..)) to the target file if the file system supports it. This enables to have objects that share the payload but have a different inode and xattrs. By default the payload-link-threshold value is G_MAXUINT64 that disables the feature. Signed-off-by: Giuseppe Scrivano Closes: #1443 Approved by: cgwalters --- man/ostree.repo-config.xml | 7 + src/libostree/ostree-repo-commit.c | 245 +++++++++++++++++++++++++- src/libostree/ostree-repo-private.h | 5 + src/libostree/ostree-repo-prune.c | 44 ++++- src/libostree/ostree-repo.c | 9 + tests/installed/itest-payload-link.sh | 86 +++++++++ 6 files changed, 388 insertions(+), 8 deletions(-) create mode 100755 tests/installed/itest-payload-link.sh diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index aa3abd54..cbc605f7 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -145,6 +145,13 @@ Boston, MA 02111-1307, USA. + + payload-link-threshold + An integer value that specifies a minimum file size for creating + a payload link. By default it is disabled. + + + diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 70068138..875743b4 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include "otutil.h" #include "ostree.h" @@ -40,6 +42,12 @@ #include "ostree-checksum-input-stream.h" #include "ostree-varint.h" +/* The standardized version of BTRFS_IOC_CLONE */ +#ifndef FICLONE +#define FICLONE _IOW(0x94, 9, int) +#endif + + /* If fsync is enabled and we're in a txn, we write into a staging dir for * commit, but we also allow direct writes into objects/ for e.g. hardlink * imports. @@ -589,6 +597,192 @@ create_regular_tmpfile_linkable_with_content (OstreeRepo *self, return TRUE; } +static gboolean +_check_support_reflink (OstreeRepo *self, gboolean *supported, GError **error) +{ + /* We have not checked yet if the file system supports reflinks, do it here */ + if (g_atomic_int_get (&self->fs_support_reflink) == 0) + { + g_auto(GLnxTmpfile) src_tmpf = { 0, }; + g_auto(GLnxTmpfile) dest_tmpf = { 0, }; + + if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_RDWR|O_CLOEXEC, + &src_tmpf, error)) + return FALSE; + if (!glnx_open_tmpfile_linkable_at (commit_tmp_dfd (self), ".", O_WRONLY|O_CLOEXEC, + &dest_tmpf, error)) + return FALSE; + + if (ioctl (dest_tmpf.fd, FICLONE, src_tmpf.fd) == 0) + g_atomic_int_set (&self->fs_support_reflink, 1); + else if (errno == EOPNOTSUPP) /* Ignore other kind of errors as they might be temporary failures */ + g_atomic_int_set (&self->fs_support_reflink, -1); + } + *supported = g_atomic_int_get (&self->fs_support_reflink) >= 0; + return TRUE; +} + +static gboolean +_create_payload_link (OstreeRepo *self, + const char *checksum, + const char *payload_checksum, + GFileInfo *file_info, + GCancellable *cancellable, + GError **error) +{ + gboolean reflinks_supported = FALSE; + if (!_check_support_reflink (self, &reflinks_supported, error)) + return FALSE; + + if (!reflinks_supported) + return TRUE; + + if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR + || !G_IN_SET(self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER, OSTREE_REPO_MODE_BARE_USER_ONLY)) + return TRUE; + + if (payload_checksum == NULL || g_file_info_get_size (file_info) < self->payload_link_threshold) + return TRUE; + + char target_buf[_OSTREE_LOOSE_PATH_MAX + _OSTREE_PAYLOAD_LINK_PREFIX_LEN]; + strcpy (target_buf, _OSTREE_PAYLOAD_LINK_PREFIX); + _ostree_loose_path (target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); + + if (symlinkat (target_buf, commit_tmp_dfd (self), payload_checksum) < 0) + { + if (errno != EEXIST) + return glnx_throw_errno_prefix (error, "symlinkat"); + } + else + { + g_auto(OtCleanupUnlinkat) tmp_unlinker = { commit_tmp_dfd (self), g_strdup (payload_checksum) }; + if (!commit_path_final (self, payload_checksum, OSTREE_OBJECT_TYPE_PAYLOAD_LINK, &tmp_unlinker, cancellable, error)) + return FALSE; + } + + return TRUE; +} + +static gboolean +_import_payload_link (OstreeRepo *self, + OstreeRepo *source, + const char *checksum, + GCancellable *cancellable, + GError **error) +{ + gboolean reflinks_supported = FALSE; + g_autofree char *payload_checksum = NULL; + g_autoptr(GInputStream) is = NULL; + glnx_unref_object OtChecksumInstream *checksum_payload = NULL; + g_autoptr(GFileInfo) file_info = NULL; + + if (!_check_support_reflink (self, &reflinks_supported, error)) + return FALSE; + + if (!reflinks_supported) + return TRUE; + + if (!G_IN_SET(self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER, OSTREE_REPO_MODE_BARE_USER_ONLY)) + return TRUE; + + if (!ostree_repo_load_file (source, checksum, &is, &file_info, NULL, cancellable, error)) + return FALSE; + + if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR + || g_file_info_get_size (file_info) < self->payload_link_threshold) + return TRUE; + + checksum_payload = ot_checksum_instream_new (is, G_CHECKSUM_SHA256); + + guint64 remaining = g_file_info_get_size (file_info); + while (remaining) + { + char buf[8192]; + gssize ret = g_input_stream_read ((GInputStream *) checksum_payload, buf, + MIN (sizeof (buf), remaining), cancellable, error); + if (ret < 0) + return FALSE; + remaining -= ret; + } + payload_checksum = ot_checksum_instream_get_string (checksum_payload); + + return _create_payload_link (self, checksum, payload_checksum, file_info, cancellable, error); +} + +static gboolean +_try_clone_from_payload_link (OstreeRepo *self, + const char *payload_checksum, + GFileInfo *file_info, + GLnxTmpfile *tmpf, + GCancellable *cancellable, + GError **error) +{ + gboolean reflinks_supported = FALSE; + int dfd_searches[] = { -1, self->objects_dir_fd }; + if (self->commit_stagedir.initialized) + dfd_searches[0] = self->commit_stagedir.fd; + + if (!_check_support_reflink (self, &reflinks_supported, error)) + return FALSE; + + if (!reflinks_supported) + return TRUE; + + for (guint i = 0; i < G_N_ELEMENTS (dfd_searches); i++) + { + glnx_autofd int fdf = -1; + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; + char loose_path_target_buf[_OSTREE_LOOSE_PATH_MAX]; + char target_buf[_OSTREE_LOOSE_PATH_MAX + _OSTREE_PAYLOAD_LINK_PREFIX_LEN]; + char target_checksum[OSTREE_SHA256_STRING_LEN+1]; + int dfd = dfd_searches[i]; + ssize_t size; + if (dfd == -1) + continue; + + _ostree_loose_path (loose_path_buf, payload_checksum, OSTREE_OBJECT_TYPE_PAYLOAD_LINK, self->mode); + + size = TEMP_FAILURE_RETRY (readlinkat (dfd, loose_path_buf, target_buf, sizeof (target_buf))); + if (size < 0) + { + if (errno == ENOENT) + continue; + return glnx_throw_errno_prefix (error, "readlinkat"); + } + + if (size < OSTREE_SHA256_STRING_LEN + _OSTREE_PAYLOAD_LINK_PREFIX_LEN) + return glnx_throw (error, "invalid data size for %s", loose_path_buf); + + sprintf (target_checksum, "%.2s%.62s", target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN, target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN + 3); + + _ostree_loose_path (loose_path_target_buf, target_checksum, OSTREE_OBJECT_TYPE_FILE, self->mode); + if (!ot_openat_ignore_enoent (dfd, loose_path_target_buf, &fdf, error)) + return FALSE; + + if (fdf < 0) + { + /* If the link is referring to an object that doesn't exist anymore in the repository, just unlink it. */ + if (!glnx_unlinkat (dfd, loose_path_buf, 0, error)) + return FALSE; + } + else + { + /* This undoes all of the previous writes; we want to generate reflinked data. */ + if (ftruncate (tmpf->fd, 0) < 0) + return glnx_throw_errno_prefix (error, "ftruncate"); + + if (glnx_regfile_copy_bytes (fdf, tmpf->fd, -1) < 0) + return glnx_throw_errno_prefix (error, "regfile copy"); + + return TRUE; + } + } + if (self->parent_repo) + return _try_clone_from_payload_link (self->parent_repo, payload_checksum, file_info, tmpf, cancellable, error); + + return TRUE; +} + /* The main driver for writing a content (regfile or symlink) object. * There are a variety of tricky cases here; for example, bare-user * repos store symlinks as regular files. Computing checksums @@ -616,6 +810,8 @@ write_content_object (OstreeRepo *self, GInputStream *file_input; /* Unowned alias */ g_autoptr(GInputStream) file_input_owned = NULL; /* We need a temporary for bare-user symlinks */ glnx_unref_object OtChecksumInstream *checksum_input = NULL; + glnx_unref_object OtChecksumInstream *checksum_payload_input = NULL; + const GFileType object_file_type = g_file_info_get_file_type (file_info); if (out_csum) { /* Previously we checksummed the input verbatim; now @@ -624,6 +820,7 @@ write_content_object (OstreeRepo *self, * it's not that's not a serious problem because we're still computing a * checksum over the data we actually use. */ + gboolean reflinks_supported = FALSE; g_autoptr(GBytes) header = _ostree_file_header_new (file_info, xattrs); size_t len; const guint8 *buf = g_bytes_get_data (header, &len); @@ -633,13 +830,26 @@ write_content_object (OstreeRepo *self, null_input = input = g_memory_input_stream_new_from_data ("", 0, NULL); checksum_input = ot_checksum_instream_new_with_start (input, G_CHECKSUM_SHA256, buf, len); - file_input = (GInputStream*)checksum_input; + + if (!_check_support_reflink (self, &reflinks_supported, error)) + return FALSE; + + if (xattrs == NULL || !G_IN_SET(self->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER, OSTREE_REPO_MODE_BARE_USER_ONLY) || object_file_type != G_FILE_TYPE_REGULAR || + !reflinks_supported) + file_input = (GInputStream*)checksum_input; + else + { + /* The payload checksum-input reads from the full object checksum-input; this + * means it skips the header. + */ + checksum_payload_input = ot_checksum_instream_new ((GInputStream*)checksum_input, G_CHECKSUM_SHA256); + file_input = (GInputStream*)checksum_payload_input; + } } else file_input = input; gboolean phys_object_is_symlink = FALSE; - const GFileType object_file_type = g_file_info_get_file_type (file_info); switch (object_file_type) { case G_FILE_TYPE_REGULAR: @@ -765,6 +975,7 @@ write_content_object (OstreeRepo *self, } const char *actual_checksum = NULL; + g_autofree char *actual_payload_checksum = NULL; g_autofree char *actual_checksum_owned = NULL; if (!checksum_input) actual_checksum = expected_checksum; @@ -777,6 +988,9 @@ write_content_object (OstreeRepo *self, error)) return FALSE; } + + if (checksum_payload_input) + actual_payload_checksum = ot_checksum_instream_get_string (checksum_payload_input); } g_assert (actual_checksum != NULL); /* Pacify static analysis */ @@ -794,6 +1008,10 @@ write_content_object (OstreeRepo *self, g_mutex_lock (&self->txn_lock); self->txn.stats.content_objects_total++; g_mutex_unlock (&self->txn_lock); + + if (!_create_payload_link (self, actual_checksum, actual_payload_checksum, file_info, cancellable, error)) + return FALSE; + if (out_csum) *out_csum = ostree_checksum_to_bytes (actual_checksum); /* Note early return */ @@ -853,12 +1071,20 @@ write_content_object (OstreeRepo *self, repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size); } + /* Check if a file with the same payload is present in the repository, + and in case try to reflink it */ + if (actual_payload_checksum && !_try_clone_from_payload_link (self, actual_payload_checksum, file_info, &tmpf, cancellable, error)) + return FALSE; + /* This path is for regular files */ if (!commit_loose_regfile_object (self, actual_checksum, &tmpf, uid, gid, mode, xattrs, cancellable, error)) return FALSE; + + if (!_create_payload_link (self, actual_checksum, actual_payload_checksum, file_info, cancellable, error)) + return FALSE; } /* Update statistics */ @@ -3999,7 +4225,11 @@ import_one_object_direct (OstreeRepo *dest_repo, if (!copy_detached_metadata (dest_repo, src_repo, checksum, cancellable, error)) return FALSE; } - + else if (objtype == OSTREE_OBJECT_TYPE_FILE) + { + if (!_import_payload_link (dest_repo, src_repo, checksum, cancellable, error)) + return FALSE; + } *out_was_supported = TRUE; return TRUE; } @@ -4092,7 +4322,14 @@ _ostree_repo_import_object (OstreeRepo *self, return FALSE; /* If we have it, we're done */ if (has_object) - return TRUE; + { + if (objtype == OSTREE_OBJECT_TYPE_FILE) + { + if (!_import_payload_link (self, source, checksum, cancellable, error)) + return FALSE; + } + return TRUE; + } if (OSTREE_OBJECT_TYPE_IS_META (objtype)) { diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index d61db28f..3078a9e2 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -53,6 +53,9 @@ G_BEGIN_DECLS #define OSTREE_SUMMARY_COLLECTION_ID "ostree.summary.collection-id" #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map" +#define _OSTREE_PAYLOAD_LINK_PREFIX "../" +#define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1) + /* Well-known keys for the additional metadata field in a commit in a ref entry * in a summary file. */ #define OSTREE_COMMIT_TIMESTAMP "ostree.commit.timestamp" @@ -161,6 +164,8 @@ struct OstreeRepo { gchar *collection_id; gboolean add_remotes_config_dir; /* Add new remotes in remotes.d dir */ gint lock_timeout_seconds; + guint64 payload_link_threshold; + gint fs_support_reflink; /* The underlying filesystem has support for ioctl (FICLONE..) */ OstreeRepo *parent_repo; }; diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index c8a178ec..f0c0a974 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -46,11 +46,14 @@ maybe_prune_loose_object (OtPruneData *data, GCancellable *cancellable, GError **error) { + gboolean reachable = FALSE; g_autoptr(GVariant) key = NULL; key = ostree_object_name_serialize (checksum, objtype); - if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) + if (g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) + reachable = TRUE; + else { guint64 storage_size = 0; @@ -65,7 +68,38 @@ maybe_prune_loose_object (OtPruneData *data, if (!(flags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)) { - if (objtype == OSTREE_OBJECT_TYPE_COMMIT) + if (objtype == OSTREE_OBJECT_TYPE_PAYLOAD_LINK) + { + ssize_t size; + char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; + char target_checksum[OSTREE_SHA256_STRING_LEN+1]; + char target_buf[_OSTREE_LOOSE_PATH_MAX + _OSTREE_PAYLOAD_LINK_PREFIX_LEN]; + + _ostree_loose_path (loose_path_buf, checksum, OSTREE_OBJECT_TYPE_PAYLOAD_LINK, data->repo->mode); + size = readlinkat (data->repo->objects_dir_fd, loose_path_buf, target_buf, sizeof (target_buf)); + if (size < 0) + return glnx_throw_errno_prefix (error, "readlinkat"); + + if (size < OSTREE_SHA256_STRING_LEN + _OSTREE_PAYLOAD_LINK_PREFIX_LEN) + return glnx_throw (error, "invalid data size for %s", loose_path_buf); + + sprintf (target_checksum, "%.2s%.62s", target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN, target_buf + _OSTREE_PAYLOAD_LINK_PREFIX_LEN + 3); + + g_autoptr(GVariant) target_key = ostree_object_name_serialize (target_checksum, OSTREE_OBJECT_TYPE_FILE); + + if (g_hash_table_lookup_extended (data->reachable, target_key, NULL, NULL)) + { + guint64 target_storage_size = 0; + if (!ostree_repo_query_object_storage_size (data->repo, OSTREE_OBJECT_TYPE_FILE, target_checksum, + &target_storage_size, cancellable, error)) + return FALSE; + + reachable = target_storage_size >= data->repo->payload_link_threshold; + if (reachable) + goto exit; + } + } + else if (objtype == OSTREE_OBJECT_TYPE_COMMIT) { if (!ostree_repo_mark_commit_partial (data->repo, checksum, FALSE, error)) return FALSE; @@ -82,7 +116,9 @@ maybe_prune_loose_object (OtPruneData *data, else data->n_unreachable_content++; } - else + + exit: + if (reachable) { g_debug ("Keeping needed object %s.%s", checksum, ostree_object_type_to_string (objtype)); @@ -286,7 +322,7 @@ repo_prune_internal (OstreeRepo *self, * of traversing all commits, only refs will be used. Particularly * when combined with @depth, this is a convenient way to delete * history from the repository. - * + * * Use the %OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE to just determine * statistics on objects that would be deleted, without actually * deleting them. diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8d94f71a..5798ae2e 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2827,6 +2827,15 @@ reload_core_config (OstreeRepo *self, return FALSE; } + { g_autofree char *payload_threshold = NULL; + + if (!ot_keyfile_get_value_with_default (self->config, "core", "payload-link-threshold", "-1", + &payload_threshold, error)) + return FALSE; + + self->payload_link_threshold = g_ascii_strtoull (payload_threshold, NULL, 10); + } + return TRUE; } diff --git a/tests/installed/itest-payload-link.sh b/tests/installed/itest-payload-link.sh new file mode 100755 index 00000000..a0764a07 --- /dev/null +++ b/tests/installed/itest-payload-link.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# +# Copyright (C) 2018 Red Hat, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libinsttest.sh + +echo "1..1" + +cd /var/srv +mkdir repo +ostree --repo=repo init --mode=archive +echo -e '[archive]\nzlib-level=1\n' >> repo/config +host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,') +ostree --repo=repo pull-local /ostree/repo ${host_commit} +ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref} + +run_tmp_webserver $(pwd)/repo + +origin=$(cat ${test_tmpdir}/httpd-address) + +cleanup() { + cd ${oldpwd} + umount mnt || true + test -n "${blkdev}" && losetup -d ${blkdev} || true + rm -rf mnt testblk.img +} +oldpwd=`pwd` +trap cleanup EXIT + +mkdir mnt +truncate -s 2G testblk.img +if ! blkdev=$(losetup --find --show $(pwd)/testblk.img); then + echo "ok # SKIP not run when cannot setup loop device" + exit 0 +fi + +mkfs.xfs -m reflink=1 ${blkdev} + +mount ${blkdev} mnt + +test_tmpdir=$(pwd)/mnt +cd ${test_tmpdir} + +touch a +if cp --reflink a b; then + mkdir repo + ostree --repo=repo init + ostree config --repo=repo set core.payload-link-threshold 0 + ostree --repo=repo remote add origin --set=gpg-verify=false ${origin} + ostree --repo=repo pull --disable-static-deltas origin ${host_nonremoteref} + if test `find repo -name '*.payload-link' | wc -l` = 0; then + fatal ".payload-link files not found" + fi + + find repo -name '*.payload-link' | while read i; + do + payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) + payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1) + if test $payload_checksum != $payload_checksum_calculated; then + fatal ".payload-link has the wrong checksum" + fi + done + echo "ok pull creates .payload-link" +else + echo "ok # SKIP no reflink support in the file system" +fi From 3b7044f45ef5f62a855454293447061016a53bd3 Mon Sep 17 00:00:00 2001 From: Jeremy Hiatt Date: Thu, 8 Mar 2018 01:44:43 +0000 Subject: [PATCH 25/51] lib/repo: Fix multi-signature support when generating summary files Ensure that the metadata object is built up with the signatures from all keys passed to ostree_repo_add_gpg_signature_summary(). Previously only the signature from the last key would end up in the metadata. Closes: #1488 Closes: #1489 Approved by: jlebon --- src/libostree/ostree-repo.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5798ae2e..8ff0d961 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4811,17 +4811,16 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, /* Note that fd is reused below */ glnx_close_fd (&fd); - g_autoptr(GVariant) existing_signatures = NULL; + g_autoptr(GVariant) metadata = NULL; if (!ot_openat_ignore_enoent (self->repo_dir_fd, "summary.sig", &fd, error)) return FALSE; if (fd != -1) { if (!ot_variant_read_fd (fd, 0, G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), - FALSE, &existing_signatures, error)) + FALSE, &metadata, error)) return FALSE; } - g_autoptr(GVariant) new_metadata = NULL; for (guint i = 0; key_id[i]; i++) { g_autoptr(GBytes) signature_data = NULL; @@ -4830,10 +4829,11 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, cancellable, error)) return FALSE; - new_metadata = _ostree_detached_metadata_append_gpg_sig (existing_signatures, signature_data); + g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata); + metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data); } - g_autoptr(GVariant) normalized = g_variant_get_normal_form (new_metadata); + g_autoptr(GVariant) normalized = g_variant_get_normal_form (metadata); if (!_ostree_repo_file_replace_contents (self, self->repo_dir_fd, From 6e9d00dbeb94d6e2248289d6d5bbb5d41de84f01 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Feb 2018 14:16:33 -0500 Subject: [PATCH 26/51] ci: Rework installed tests to use Fedora Standard Test interface Reusing the way `standard-test-roles` has support for booting a qcow2 actually gets us to the "VM-in-container" flow. Plus Ansible over shell script is sometimes nicer. https://fedoraproject.org/wiki/CI/Tests#Testing_an_Atomic_Host It's better than what we were doing before for installed tests, and moreover using Ansible more broadly for testing is going to align us better with Fedora's CI. As part of this I split off a "libpaprci" which I intend to maintain as a "copylib" for a little bit between ostree/rpm-ostree, and then we'll figure out how to expand from there (maybe some of the patterns get "baked in" to PAPR for example). Note the `FAH27-insttests` context moves to the top since it's now of primary importance, and I expect that we start expanding it. Closes: #1462 Approved by: jlebon --- .papr.yml | 88 ++++++++------------------- ci/build-check.sh | 27 ++++---- ci/build-rpm.sh | 46 ++++++++++++++ ci/build.sh | 19 +++--- ci/ci-commitmessage-submodules.sh | 2 +- ci/flatpak.sh | 6 +- ci/libbuild.sh | 69 --------------------- ci/libpaprci/Makefile.dist-packaging | 39 ++++++++++++ ci/libpaprci/libbuild.sh | 74 ++++++++++++++++++++++ ci/libpaprci/make-git-snapshot.sh | 32 ++++++++++ ci/libpaprci/rpmbuild-cwd | 11 ++++ ci/provision-prep.sh | 16 +++++ ci/rpmostree.sh | 6 +- tests/fedora-str/README.md | 2 + tests/fedora-str/overlay-git.yml | 31 ++++++++++ tests/fedora-str/provision.sh | 8 +++ tests/fedora-str/run.sh | 25 ++++++++ tests/fedora-str/sysinstall-tests.yml | 20 ++++++ tests/fedora-str/tests.yml | 2 + tests/installed/fah-prep.sh | 10 --- 20 files changed, 361 insertions(+), 172 deletions(-) create mode 100755 ci/build-rpm.sh delete mode 100644 ci/libbuild.sh create mode 100644 ci/libpaprci/Makefile.dist-packaging create mode 100644 ci/libpaprci/libbuild.sh create mode 100755 ci/libpaprci/make-git-snapshot.sh create mode 100755 ci/libpaprci/rpmbuild-cwd create mode 100755 ci/provision-prep.sh create mode 100644 tests/fedora-str/README.md create mode 100644 tests/fedora-str/overlay-git.yml create mode 100755 tests/fedora-str/provision.sh create mode 100755 tests/fedora-str/run.sh create mode 100644 tests/fedora-str/sysinstall-tests.yml create mode 100644 tests/fedora-str/tests.yml delete mode 100755 tests/installed/fah-prep.sh diff --git a/.papr.yml b/.papr.yml index 6e67d232..bb84e689 100644 --- a/.papr.yml +++ b/.papr.yml @@ -1,54 +1,45 @@ +# https://fedoraproject.org/wiki/CI/Tests branches: - master - auto - try +context: FAH27-insttests required: true -context: f27-primary container: - image: registry.fedoraproject.org/fedora:27 + image: registry.fedoraproject.org/fedora:27 +tests: + - cd tests/fedora-str && ../../ci/build-rpm.sh + - ./tests/fedora-str/provision.sh + # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/ + - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest + - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/run.sh + +artifacts: + - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log + - tests/fedora-str/artifacts/installed-tests.log + +--- + +# This suite skips the RPMs and does the build+unit tests in a container +inherit: true +context: f27-primary env: - # Enable all the sanitizers for this primary build. - # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain - CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' - # Only for CI with a known g-ir-scanner - GI_SCANNERFLAGS: '--warn-error' - ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc - # TODO when we're doing leak checks: G_SLICE: "always-malloc" - CONFIGOPTS: '--with-curl --with-openssl' + # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -fsanitize=address -O2 -Wp,-D_FORTIFY_SOURCE=2' + # Only for CI with a known g-ir-scanner + GI_SCANNERFLAGS: '--warn-error' + ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc + # TODO when we're doing leak checks: G_SLICE: "always-malloc" + CONFIGOPTS: '--with-curl --with-openssl' tests: - ci/ci-commitmessage-submodules.sh - ci/build-check.sh - ci/ci-release-build.sh -timeout: 30m - -# Keep this in sync with build-check.sh -artifacts: - - test-suite.log - - config.log - - gdtr-results ---- - -context: c7-primary -inherit: true -required: true - -host: - distro: centos/7/atomic - -env: - CFLAGS: '' - CONFIGOPTS: '--with-curl --with-openssl' - -tests: - - docker run --privileged -v $PWD:$PWD --workdir $PWD - registry.centos.org/centos/centos:7 sh -c - 'yum install -y git && ci/build-check.sh' - --- context: f27-rust @@ -132,33 +123,6 @@ tests: --- -inherit: false -branches: - - master - - auto - - try - -context: f27ah-insttest -required: false - -cluster: - hosts: - - name: vmcheck - distro: fedora/27/atomic - container: - image: registry.fedoraproject.org/fedora:27 - -# Copy the build from the container to the host; ideally down the line -# this is installing an RPM via https://github.com/jlebon/redhat-ci/issues/10 -tests: - - ci/build.sh - - make install DESTDIR=$(pwd)/insttree - - yum -y install rsync - - rsync -rl -e 'ssh -o User=root' . vmcheck:ostree/ - - ssh root@vmcheck './ostree/tests/installed/fah-prep.sh && ./ostree/tests/installed/run.sh' - ---- - inherit: false branches: - master diff --git a/ci/build-check.sh b/ci/build-check.sh index 6e8b3930..c09e9a52 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -4,7 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libbuild.sh +. ${dn}/libpaprci/libbuild.sh ${dn}/build.sh topdir=$(git rev-parse --show-toplevel) resultsdir=$(mktemp -d) @@ -14,9 +14,21 @@ make syntax-check # TODO: do syntax-check under check for x in test-suite.log config.log; do mv ${x} ${resultsdir} done -# And now run the installed tests +# And now install; we'll run the test suite after we do a clang build first +# (But we don't install that one) make install +# And now a clang build to find unused variables because it does a better +# job than gcc for vars with cleanups; perhaps in the future these could +# parallelize +if test -x /usr/bin/clang; then + # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument] + export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}" + git clean -dfx && git submodule foreach git clean -dfx + export CC=clang + build +fi + copy_out_gdtr_artifacts() { # Keep this in sync with papr.yml # TODO; Split the main/clang builds into separate build dirs @@ -40,14 +52,3 @@ if test -x /usr/bin/gnome-desktop-testing-runner; then # Use the new -L option gnome-desktop-testing-runner -L ${resultsdir}/gdtr-results -p 0 ${INSTALLED_TESTS_PATTERN:-libostree/} fi - -# And now a clang build to find unused variables because it does a better -# job than gcc for vars with cleanups; perhaps in the future these could -# parallelize -if test -x /usr/bin/clang; then - # Except for clang-4.0: error: argument unused during compilation: '-specs=/usr/lib/rpm/redhat/redhat-hardened-cc1' [-Werror,-Wunused-command-line-argument] - export CFLAGS="-Wall -Werror -Wno-error=unused-command-line-argument ${CFLAGS:-}" - git clean -dfx && git submodule foreach git clean -dfx - export CC=clang - build -fi diff --git a/ci/build-rpm.sh b/ci/build-rpm.sh new file mode 100755 index 00000000..5d096333 --- /dev/null +++ b/ci/build-rpm.sh @@ -0,0 +1,46 @@ +#!/usr/bin/bash +# Generate a src.rpm, then binary rpms in the current directory + +set -xeuo pipefail + +dn=$(dirname $0) +paprcidir=${dn}/libpaprci +. ${paprcidir}/libbuild.sh + +# Auto-provision bootstrap resources if run as root (normally in CI) +if test "$(id -u)" == 0; then + pkg_install_buildroot + pkg_install make /usr/bin/rpmbuild git +fi + +# PAPR really should do this +if ! test -f libglnx/README.md || ! test -f bsdiff/README.md; then + git submodule update --init +fi + +# Default libcurl on by default in fedora unless libsoup is enabled +if test "${OS_ID}" = 'fedora'; then + case "${CONFIGOPTS:-}" in + *--with-soup*|*--without-curl*) ;; + *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl" + esac +fi +case "${CONFIGOPTS:-}" in + *--with-curl*|*--with-soup*) + if test -x /usr/bin/gnome-desktop-testing-runner; then + CONFIGOPTS="${CONFIGOPTS} --enable-installed-tests=exclusive" + fi + ;; +esac + +# TODO: Use some form of rpm's --build-in-place to skip archive-then-unpack? +make -f ${paprcidir}/Makefile.dist-packaging srpm PACKAGE=libostree DISTGIT_NAME=ostree +if test "$(id -u)" == 0; then + pkg_builddep *.src.rpm +else + echo "NOTE: Running as non-root, assuming build dependencies are installed" +fi +if ! ${paprcidir}/rpmbuild-cwd --rebuild *.src.rpm; then + find . -type f -name config.log -exec cat {} \; + exit 1 +fi diff --git a/ci/build.sh b/ci/build.sh index be05604a..4f7d02c4 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -4,12 +4,11 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libbuild.sh +. ${dn}/libpaprci/libbuild.sh pkg_upgrade -pkg_install_builddeps ostree -# Until this propagates farther -pkg_install 'pkgconfig(libcurl)' 'pkgconfig(openssl)' +pkg_install_buildroot +pkg_builddep ostree pkg_install sudo which attr fuse strace \ libubsan libasan libtsan PyYAML redhat-rpm-config \ elfutils @@ -18,15 +17,13 @@ if test -n "${CI_PKGS:-}"; then fi pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang \ python3-PyYAML -(. /etc/os-release; - if test "${ID}" = "centos"; then - rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - pkg_install python34{,-PyYAML} - fi -) +if test "${OS_ID}" = "centos"; then + rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + pkg_install python34{,-PyYAML} +fi # Default libcurl on by default in fedora unless libsoup is enabled -if sh -c '. /etc/os-release; test "${ID}" = fedora'; then +if test "${OS_ID}" = 'fedora'; then case "${CONFIGOPTS:-}" in *--with-soup*|*--without-curl*) ;; *) CONFIGOPTS="${CONFIGOPTS:-} --with-curl" diff --git a/ci/ci-commitmessage-submodules.sh b/ci/ci-commitmessage-submodules.sh index 2dc9b764..38481aaf 100755 --- a/ci/ci-commitmessage-submodules.sh +++ b/ci/ci-commitmessage-submodules.sh @@ -17,7 +17,7 @@ set -euo pipefail # being tested rather than the merge sha HEAD=${PAPR_COMMIT:-HEAD} dn=$(dirname $0) -. ${dn}/libbuild.sh +. ${dn}/libpaprci/libbuild.sh tmpd=$(mktemp -d) touch ${tmpd}/.tmpdir diff --git a/ci/flatpak.sh b/ci/flatpak.sh index cdd03593..015cd390 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -9,13 +9,13 @@ set -xeuo pipefail FLATPAK_TAG=0.10.2.1 dn=$(dirname $0) -. ${dn}/libbuild.sh +. ${dn}/libpaprci/libbuild.sh codedir=$(pwd) pkg_upgrade -pkg_install_builddeps ostree -pkg_install_builddeps flatpak +pkg_install_buildroot +pkg_builddep ostree flatpak pkg_install gettext-devel # A new dependency # Copy of builddeps from build.sh in flatpak pkg_install sudo which attr fuse \ diff --git a/ci/libbuild.sh b/ci/libbuild.sh deleted file mode 100644 index 2b600dc7..00000000 --- a/ci/libbuild.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/bash - -pkg_upgrade() { - # https://bugzilla.redhat.com/show_bug.cgi?id=1483553 - ecode=0 - yum -y distro-sync 2>err.txt || ecode=$? - if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then - rpm --rebuilddb - yum -y distro-sync - else - if test ${ecode} '!=' 0; then - cat err.txt - exit ${ecode} - fi - fi -} - -make() { - /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@" -} - -build() { - env NOCONFIGURE=1 ./autogen.sh - ./configure --prefix=/usr --libdir=/usr/lib64 "$@" - make V=1 -} - -pkg_install() { - yum -y install "$@" -} - -pkg_install_if_os() { - os=$1 - shift - (. /etc/os-release; - if test "${os}" = "${ID}"; then - pkg_install "$@" - else - echo "Skipping installation on OS ${ID}: $@" - fi - ) -} - -pkg_builddep() { - # This is sadly the only case where it's a different command - if test -x /usr/bin/dnf; then - dnf builddep -y "$@" - else - yum-builddep -y "$@" - fi -} - -pkg_install_builddeps() { - pkg=$1 - if test -x /usr/bin/dnf; then - yum -y install dnf-plugins-core - yum install -y 'dnf-command(builddep)' - # Base buildroot - pkg_install @buildsys-build - else - yum -y install yum-utils - # Base buildroot, copied from the mock config sadly - yum -y install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz - fi - # builddeps+runtime deps - pkg_builddep $pkg - pkg_install $pkg - rpm -e $pkg -} diff --git a/ci/libpaprci/Makefile.dist-packaging b/ci/libpaprci/Makefile.dist-packaging new file mode 100644 index 00000000..8ecaeffb --- /dev/null +++ b/ci/libpaprci/Makefile.dist-packaging @@ -0,0 +1,39 @@ +# -*- mode: Makefile -*- + +mypath = $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +topsrcdir = $(shell git rev-parse --show-toplevel) +GITREV = $(shell git describe --always --tags) +GITREV_FOR_PKG = $(shell echo "$(GITREV)" | sed -e 's,-,\.,g' -e 's,^v,,') + +PACKAGE ?= $(shell basename $(topsrcdir)) +DISTGIT_NAME ?= $(PACKAGE) +DISTGIT ?= https://src.fedoraproject.org/rpms/$(DISTGIT_NAME) +SPEC ?= $(topsrcdir)/$(DISTGIT_NAME).spec + +PKG_VER = $(PACKAGE)-$(GITREV_FOR_PKG) +PKG_CLIENT_VER = $(PACKAGE)-client-$(GITREV_FOR_PKG) + +dist-snapshot: + if ! test -f $(PKG_VER).tar.xz; then \ + $(mypath)/make-git-snapshot.sh "$(topsrcdir)" "$(PKG_VER)" "$(GITREV)" && \ + rm -f $(PKG_VER).tar.xz && \ + xz $(PKG_VER).tar; \ + fi + +srpm: dist-snapshot + if test -f "$(SPEC)"; then \ + sed -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(SPEC) > $(DISTGIT_NAME).spec && \ + $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec ; \ + else \ + test -d $(DISTGIT_NAME) || git clone --depth=1 $(DISTGIT) && \ + mv $(PKG_VER).tar.xz $(DISTGIT_NAME) && \ + origdir=$$(pwd); \ + cd $(DISTGIT_NAME) && \ + git stash && git pull -r && \ + sed -i -e "s,^Version:.*,Version: $(GITREV_FOR_PKG)," $(DISTGIT_NAME).spec && \ + rm -f *.src.rpm && \ + $(mypath)/rpmbuild-cwd -bs $(DISTGIT_NAME).spec && mv *.src.rpm $${origdir}; \ + fi + +rpm: srpm + ./rpmbuild-cwd --rebuild $(PKG_VER)*.src.rpm diff --git a/ci/libpaprci/libbuild.sh b/ci/libpaprci/libbuild.sh new file mode 100644 index 00000000..074494f0 --- /dev/null +++ b/ci/libpaprci/libbuild.sh @@ -0,0 +1,74 @@ +#!/usr/bin/bash + +dn=$(cd $(dirname $0) && pwd) + +OS_ID=$(. /etc/os-release; echo $ID) +OS_VERSION_ID=$(. /etc/os-release; echo $VERSION_ID) + +pkg_upgrade() { + # https://bugzilla.redhat.com/show_bug.cgi?id=1483553 + local ecode=0 + yum -y distro-sync 2>err.txt || ecode=$? + if test ${ecode} '!=' 0 && grep -q -F -e "BDB1539 Build signature doesn't match environment" err.txt; then + rpm --rebuilddb + yum -y distro-sync + else + if test ${ecode} '!=' 0; then + cat err.txt + exit ${ecode} + fi + fi +} + +make() { + /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@" +} + +build() { + env NOCONFIGURE=1 ./autogen.sh + ./configure --sysconfdir=/etc --prefix=/usr --libdir=/usr/lib64 "$@" + make V=1 +} + +pkg_install() { + yum -y install "$@" +} + +pkg_install_if_os() { + local os=$1 + shift + if test "${os}" = "${OS_ID}"; then + pkg_install "$@" + else + echo "Skipping installation targeted for ${os} (current OS: ${OS_ID}): $@" + fi +} + +pkg_install_buildroot() { + case "${OS_ID}" in + fedora) pkg_install dnf-plugins-core @buildsys-build;; + centos) pkg_install yum-utils + # Base buildroot, copied from the mock config sadly + pkg_install bash bzip2 coreutils cpio diffutils system-release findutils gawk gcc gcc-c++ \ + grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar \ + unzip util-linux which xz;; + *) fatal "pkg_install_buildroot(): Unhandled OS ${OS_ID}";; + esac +} + +pkg_builddep() { + # This is sadly the only case where it's a different command + if test -x /usr/bin/dnf; then + dnf builddep -y "$@" + else + yum-builddep -y "$@" + fi +} + +# Install both build and runtime dependencies for $pkg +pkg_builddep_runtimedep() { + local pkg=$1 + pkg_builddep $pkg + pkg_install $pkg + rpm -e $pkg +} diff --git a/ci/libpaprci/make-git-snapshot.sh b/ci/libpaprci/make-git-snapshot.sh new file mode 100755 index 00000000..1a137bff --- /dev/null +++ b/ci/libpaprci/make-git-snapshot.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +srcdir=$1 +shift +PKG_VER=$1 +shift +GITREV=$1 +shift + +TARFILE=${PKG_VER}.tar +TARFILE_TMP=${TARFILE}.tmp + +set -x +set -e + +test -n "${srcdir}" +test -n "${PKG_VER}" +test -n "${GITREV}" + +TOP=$(git rev-parse --show-toplevel) + +echo "Archiving ${PKG_VER} at ${GITREV} to ${TARFILE_TMP}" +(cd ${TOP}; git archive --format=tar --prefix=${PKG_VER}/ ${GITREV}) > ${TARFILE_TMP} +ls -al ${TARFILE_TMP} +(cd ${TOP}; git submodule status) | while read line; do + rev=$(echo ${line} | cut -f 1 -d ' '); path=$(echo ${line} | cut -f 2 -d ' ') + echo "Archiving ${path} at ${rev}" + (cd ${srcdir}/${path}; git archive --format=tar --prefix=${PKG_VER}/${path}/ ${rev}) > submodule.tar + tar -A -f ${TARFILE_TMP} submodule.tar + rm submodule.tar +done +mv ${TARFILE_TMP} ${TARFILE} diff --git a/ci/libpaprci/rpmbuild-cwd b/ci/libpaprci/rpmbuild-cwd new file mode 100755 index 00000000..d0805bb5 --- /dev/null +++ b/ci/libpaprci/rpmbuild-cwd @@ -0,0 +1,11 @@ +#!/bin/sh +# rpmbuild-cwd: +# Run "rpmbuild", defining all RPM variables to use the current directory. +# This matches Fedora's system. +# +# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) +# Copyright (C) 2010 Red Hat, Inc. +# Written by Colin Walters + +pwd=$(pwd) +exec rpmbuild --define "_sourcedir ${pwd}" --define "_specdir ${pwd}" --define "_builddir ${pwd}" --define "_srcrpmdir ${pwd}" --define "_rpmdir ${pwd}" --define "_buildrootdir ${pwd}/.build" "$@" diff --git a/ci/provision-prep.sh b/ci/provision-prep.sh new file mode 100755 index 00000000..30825802 --- /dev/null +++ b/ci/provision-prep.sh @@ -0,0 +1,16 @@ +#!/usr/bin/bash +# Prepare the current environment + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libpaprci/libbuild.sh +pkg_upgrade +pkg_install_buildroot +pkg_install sudo which attr fuse strace \ + libubsan libasan libtsan PyYAML elfutils +pkg_install_if_os fedora gjs gnome-desktop-testing parallel coccinelle clang + +if test -n "${CI_PKGS:-}"; then + pkg_install ${CI_PKGS} +fi diff --git a/ci/rpmostree.sh b/ci/rpmostree.sh index d40dd6a2..e8189c40 100755 --- a/ci/rpmostree.sh +++ b/ci/rpmostree.sh @@ -9,13 +9,13 @@ set -xeuo pipefail RPMOSTREE_TAG=v2017.11 dn=$(dirname $0) -. ${dn}/libbuild.sh +. ${dn}/libpaprci/libbuild.sh codedir=$(pwd) pkg_upgrade -pkg_install_builddeps ostree -pkg_install_builddeps rpm-ostree +pkg_install_buildroot +pkg_builddep ostree rpm-ostree pkg_install rpm-ostree && rpm -e rpm-ostree # Duplicate of deps from build.sh in rpm-ostree for tests diff --git a/tests/fedora-str/README.md b/tests/fedora-str/README.md new file mode 100644 index 00000000..8f219380 --- /dev/null +++ b/tests/fedora-str/README.md @@ -0,0 +1,2 @@ +This directory holds tests that use the +[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface). diff --git a/tests/fedora-str/overlay-git.yml b/tests/fedora-str/overlay-git.yml new file mode 100644 index 00000000..41d12846 --- /dev/null +++ b/tests/fedora-str/overlay-git.yml @@ -0,0 +1,31 @@ +--- +- hosts: localhost + tags: + - atomic + remote_user: root + tasks: + - command: ostree --version + changed_when: False + register: ostree_orig_version + - set_fact: + ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}" + - name: Copy locally built RPMs + synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes + - shell: ostree admin unlock || true + # Install the RPMs we already have. For the test suite we use rpm2cpio + # since it depends on libsoup, but we're not using that yet for the sysinstalled tests + - shell: > + /usr/bin/rpm -Fvh /root/x86_64/*.rpm && \ + cd / && rpm2cpio /root/x86_64/ostree-tests-2*.rpm | cpio -div + - command: ostree --version + register: ostree_new_version + - set_fact: + ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}" + - name: "Fail if we didn't change the ostree version" + when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] + fail: + msg: "Failed to change ostree version" + + # Next copy all of the tests/ directory + - name: Copy test data + synchronize: src=../../ dest=/root/tests/ archive=yes diff --git a/tests/fedora-str/provision.sh b/tests/fedora-str/provision.sh new file mode 100755 index 00000000..b8ecdb48 --- /dev/null +++ b/tests/fedora-str/provision.sh @@ -0,0 +1,8 @@ +#!/usr/bin/bash +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/../../ci/libpaprci/libbuild.sh + +pkg_upgrade +pkg_install git rsync openssh-clients ansible standard-test-roles diff --git a/tests/fedora-str/run.sh b/tests/fedora-str/run.sh new file mode 100755 index 00000000..8714a85b --- /dev/null +++ b/tests/fedora-str/run.sh @@ -0,0 +1,25 @@ +#!/usr/bin/bash +set -xeuo pipefail + +dn=$(cd $(dirname $0) && pwd) +cd ${dn} + +# https://fedoraproject.org/wiki/CI/Tests +if test -z "${TEST_SUBJECTS:-}"; then + cat < /root/installed-tests.log + register: sysinstalled_result + failed_when: False + - name: Fetch sysinstalled results + fetch: + src: /root/installed-tests.log + dest: artifacts/installed-tests.log + flat: yes + - name: Assert that sysinstalled tests succeeded + when: sysinstalled_result.rc != 0 + fail: + msg: "sysinstalled tests failed" diff --git a/tests/fedora-str/tests.yml b/tests/fedora-str/tests.yml new file mode 100644 index 00000000..7a78adb3 --- /dev/null +++ b/tests/fedora-str/tests.yml @@ -0,0 +1,2 @@ +- include: overlay-git.yml +- include: sysinstall-tests.yml diff --git a/tests/installed/fah-prep.sh b/tests/installed/fah-prep.sh deleted file mode 100755 index 865fa4f1..00000000 --- a/tests/installed/fah-prep.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -xeuo pipefail -# If we're using devmapper, expand the root -if lvm lvs atomicos/docker-pool &>/dev/null; then - systemctl stop docker - lvm lvremove -f atomicos/docker-pool -fi -lvm lvextend -r -l +100%FREE atomicos/root -ostree admin unlock -rsync -rlv ./ostree/insttree/usr/ /usr/ From 591f8a68b188e7e9239532e93ed452768589e281 Mon Sep 17 00:00:00 2001 From: Joaquim Rocha Date: Mon, 12 Mar 2018 13:24:22 +0100 Subject: [PATCH 27/51] pull: Ignore the cancellable when aborting a transaction In ostree_repo_abort_transaction, if we pass a cancellable and it gets canceled, then the function may fail to fully clean up the transaction state. This was happening e.g. when the ostree_repo_pull_with_options call got cancelled. To fix this, as suggested by Colin Walters, we set the passed cancellable as NULL, in order for it to be ignored. https://github.com/ostreedev/ostree/issues/1491 Closes: #1492 Approved by: jlebon --- src/libostree/ostree-repo-commit.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 875743b4..16081a95 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2153,6 +2153,12 @@ ostree_repo_abort_transaction (OstreeRepo *self, GCancellable *cancellable, GError **error) { + /* Always ignore the cancellable to avoid the chance that, if it gets + * canceled, the transaction may not be fully cleaned up. + * See https://github.com/ostreedev/ostree/issues/1491 . + */ + cancellable = NULL; + /* Note early return */ if (!self->in_transaction) return TRUE; From 79fc506284bb4bfa7c8434a52f21229d05268790 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Mar 2018 18:02:04 -0400 Subject: [PATCH 28/51] tests: Avoid generating lots of output in itest-payload-link We noticed this in a recent PR. While I'm here, also only do the `find` once, add `-type l` for good measure, and use our built in `libtest.sh` assertion functions. Closes: #1494 Approved by: giuseppe --- tests/installed/itest-payload-link.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/installed/itest-payload-link.sh b/tests/installed/itest-payload-link.sh index a0764a07..807d497d 100755 --- a/tests/installed/itest-payload-link.sh +++ b/tests/installed/itest-payload-link.sh @@ -68,18 +68,17 @@ if cp --reflink a b; then ostree config --repo=repo set core.payload-link-threshold 0 ostree --repo=repo remote add origin --set=gpg-verify=false ${origin} ostree --repo=repo pull --disable-static-deltas origin ${host_nonremoteref} - if test `find repo -name '*.payload-link' | wc -l` = 0; then - fatal ".payload-link files not found" - fi + find repo -type l -name '*.payload-link' >payload-links.txt + assert_not_streq "$(wc -l < payload-links.txt)" "0" - find repo -name '*.payload-link' | while read i; - do + # Disable logging for inner loop, otherwise it'd be enormous + set +x + cat payload-links.txt | while read i; do payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1) - if test $payload_checksum != $payload_checksum_calculated; then - fatal ".payload-link has the wrong checksum" - fi + assert_streq "${payload_checksum}" "${payload_checksum_calculated}" done + set -x echo "ok pull creates .payload-link" else echo "ok # SKIP no reflink support in the file system" From 17db0f15a79835b76ede6785120d237066c57d32 Mon Sep 17 00:00:00 2001 From: Rasmus Thomsen Date: Thu, 8 Mar 2018 17:20:11 +0100 Subject: [PATCH 29/51] configure: add option for libsystemd Until now ostree checked for libsystemd and enabled support for it if it found it. This commit changes that behavior by adding an option to enable/disable libsystemd. This is especially useful if one uses a source based distro (like Gentoo/Exherbo), where one wants to avoid such automagic detection of dependencies and prefers switches for that instead. Closes: #1490 Approved by: cgwalters --- configure.ac | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 3b2ff26f..3547f4f3 100644 --- a/configure.ac +++ b/configure.ac @@ -469,10 +469,26 @@ AC_ARG_WITH(mkinitcpio, AM_CONDITIONAL(BUILDOPT_MKINITCPIO, test x$with_mkinitcpio = xyes) dnl We have separate checks for libsystemd and the unit dir for historical reasons -PKG_CHECK_MODULES([LIBSYSTEMD], [libsystemd], [have_libsystemd=yes], [have_libsystemd=no]) -AM_CONDITIONAL(BUILDOPT_LIBSYSTEMD, test x$have_libsystemd = xyes) -AM_COND_IF(BUILDOPT_LIBSYSTEMD, - AC_DEFINE([HAVE_LIBSYSTEMD], 1, [Define if we have libsystemd])) +AC_ARG_WITH(libsystemd, + AS_HELP_STRING([--without-libsystemd], [Do not use libsystemd]), + :, with_libsystemd=maybe) + +AS_IF([ test x$with_libsystemd != xno ], [ + AC_MSG_CHECKING([for libsystemd]) + PKG_CHECK_EXISTS(libsystemd, have_libsystemd=yes, have_libsystemd=no) + AC_MSG_RESULT([$have_libsystemd]) + AS_IF([ test x$have_libsystemd = xno && test x$with_libsystemd != xmaybe ], [ + AC_MSG_ERROR([libsystemd is enabled but could not be found]) + ]) + AS_IF([ test x$have_libsystemd = xyes], [ + AC_DEFINE([HAVE_LIBSYSTEMD], 1, [Define if we have libsystemd.pc]) + PKG_CHECK_MODULES([LIBSYSTEMD], [libsystemd]) + with_libsystemd=yes + ], [ + with_libsystemd=no + ]) +], [ with_libsystemd=no ]) +AM_CONDITIONAL(BUILDOPT_LIBSYSTEMD, test $with_libsystemd != no) AS_IF([test "x$have_libsystemd" = "xyes"], [ with_systemd=yes From 792c190a445ae33e46812910a694c4c111b87572 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Mar 2018 13:16:43 -0400 Subject: [PATCH 30/51] lib/deploy: Port deployment checkout func to new style Not sure how we missed this one before. No functional changes, just prep for further work. Closes: #1497 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 44 +++++++++++---------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 8fdf28a3..f0a667d7 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -523,38 +523,33 @@ checkout_deployment_tree (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - OstreeRepoCheckoutAtOptions checkout_opts = { 0, }; - const char *csum = ostree_deployment_get_csum (deployment); - g_autofree char *checkout_target_name = NULL; - g_autofree char *osdeploy_path = NULL; - glnx_autofd int osdeploy_dfd = -1; - int ret_fd; - - osdeploy_path = g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL); - checkout_target_name = g_strdup_printf ("%s.%d", csum, ostree_deployment_get_deployserial (deployment)); - + GLNX_AUTO_PREFIX_ERROR ("Checking out deployment tree", error); + /* Find the directory with deployments for this stateroot */ + g_autofree char *osdeploy_path = + g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL); if (!glnx_shutil_mkdir_p_at (sysroot->sysroot_fd, osdeploy_path, 0775, cancellable, error)) - goto out; + return FALSE; + glnx_autofd int osdeploy_dfd = -1; if (!glnx_opendirat (sysroot->sysroot_fd, osdeploy_path, TRUE, &osdeploy_dfd, error)) - goto out; + return FALSE; + /* Clean up anything that was there before, from e.g. an interrupted checkout */ + const char *csum = ostree_deployment_get_csum (deployment); + g_autofree char *checkout_target_name = + g_strdup_printf ("%s.%d", csum, ostree_deployment_get_deployserial (deployment)); if (!glnx_shutil_rm_rf_at (osdeploy_dfd, checkout_target_name, cancellable, error)) - goto out; + return FALSE; + /* Generate hardlink farm, then opendir it */ + OstreeRepoCheckoutAtOptions checkout_opts = { 0, }; if (!ostree_repo_checkout_at (repo, &checkout_opts, osdeploy_dfd, checkout_target_name, csum, cancellable, error)) - goto out; + return FALSE; - if (!glnx_opendirat (osdeploy_dfd, checkout_target_name, TRUE, &ret_fd, error)) - goto out; - - ret = TRUE; - *out_deployment_dfd = ret_fd; - out: - return ret; + return glnx_opendirat (osdeploy_dfd, checkout_target_name, TRUE, out_deployment_dfd, + error); } static char * @@ -2446,10 +2441,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, glnx_autofd int deployment_dfd = -1; if (!checkout_deployment_tree (self, repo, new_deployment, &deployment_dfd, cancellable, error)) - { - g_prefix_error (error, "Checking out tree: "); - return FALSE; - } + return FALSE; g_autoptr(OstreeKernelLayout) kernel_layout = NULL; if (!get_kernel_from_tree (deployment_dfd, &kernel_layout, From bb9cc8912a0208a1b383ddbd24ecde6c7453e730 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Mar 2018 13:24:09 -0400 Subject: [PATCH 31/51] sysroot: Track whether /run/ostree-booted exists Prep for further work around deployment staging. Closes: #1497 Approved by: jlebon --- src/libostree/ostree-sysroot-private.h | 1 + src/libostree/ostree-sysroot.c | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 3ed6237b..cb4aac9d 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -50,6 +50,7 @@ struct OstreeSysroot { GLnxLockFile lock; gboolean loaded; + gboolean ostree_booted; gboolean is_physical; /* TRUE if we're pointed at physical storage root and not a deployment */ GPtrArray *deployments; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 2d12deb6..3294e83d 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -797,6 +797,16 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!ensure_repo (self, error)) return FALSE; + /* If we didn't check already, see if we have the global ostree-booted flag; + * we'll use it to sanity check that we found a booted deployment for example. + */ + if (!self->loaded) + { + if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) + return FALSE; + self->ostree_booted = (errno == 0); + } + int bootversion = 0; if (!read_current_bootversion (self, &bootversion, cancellable, error)) return FALSE; @@ -852,6 +862,9 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!find_booted_deployment (self, deployments, &self->booted_deployment, cancellable, error)) return FALSE; + /* Sanity check; note the converse case is fine */ + if (self->booted_deployment && !self->ostree_booted) + return glnx_throw (error, "Unexpected state: In a booted deployment but no /run/ostree-booted?"); /* Determine whether we're "physical" or not, the first time we initialize */ if (!self->loaded) From 5b3f79d4bb7dc80f4bc750a9e8911709646a5523 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 12 Mar 2018 13:55:51 -0500 Subject: [PATCH 32/51] sysroot: Rework how we find booted deployment I was looking at this code in prep for "staging" deployments, and there are several cleanups to be made here. The first thing I noticed is that we look for the `ostree=` kernel argument, but the presence of that should be exactly equivalent to having `/run/ostree-booted` exist. We just added a member variable for that, so let's make use of it. Related to this, we were erroring out if we had the karg but didn't find a deployment. But this can happen if e.g. one is using `ostree admin --sysroot` from an ostree-booted system! It's actually a bit surprising no one has reported this so far; I guess in the end people are either using non-ostree systems or running from containers. Let's add a member variable `root_is_sysroot` that we can use to determine if we're looking at `/`. Then, our more precise "should find a booted deployment" state is when both `ostree_booted` and `root_is_sysroot` are TRUE. Next, rather than walking all of the deployments after parsing, we can inline the `fstatat()` while parsing. The mild ugly thing about this is assigning to the sysroot member variable while parsing, but I will likely clean that up later, just wanted to avoid rewriting everything in one go. Closes: #1497 Approved by: jlebon --- src/libostree/ostree-sysroot-private.h | 4 + src/libostree/ostree-sysroot.c | 140 ++++++++----------------- 2 files changed, 50 insertions(+), 94 deletions(-) diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index cb4aac9d..b2776e8d 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -51,6 +51,10 @@ struct OstreeSysroot { gboolean loaded; gboolean ostree_booted; + gboolean root_is_sysroot; /* TRUE if sysroot_fd is pointed to rootfs "/" */ + /* The device/inode for /, used to detect booted deployment */ + dev_t root_device; + ino_t root_inode; gboolean is_physical; /* TRUE if we're pointed at physical storage root and not a deployment */ GPtrArray *deployments; diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 3294e83d..1cf7779a 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -37,13 +37,6 @@ #include "ostree-bootloader-syslinux.h" #include "ostree-bootloader-grub2.h" -static gboolean -find_booted_deployment (OstreeSysroot *self, - GPtrArray *deployments, - OstreeDeployment **out_deployment, - GCancellable *cancellable, - GError **error); - /** * SECTION:ostree-sysroot * @title: Root partition mount point @@ -642,6 +635,24 @@ parse_deployment (OstreeSysroot *self, &deployment_dfd, error)) return FALSE; + /* See if this is the booted deployment */ + const gboolean looking_for_booted_deployment = + (self->ostree_booted && self->root_is_sysroot && + !self->booted_deployment); + gboolean is_booted_deployment = FALSE; + if (looking_for_booted_deployment) + { + struct stat stbuf; + if (!glnx_fstat (deployment_dfd, &stbuf, error)) + return FALSE; + /* A bit ugly, we're assigning to a sysroot-owned variable from deep in + * this parsing code. But eh, if something fails the sysroot state can't + * be relied on anyways. + */ + is_booted_deployment = (stbuf.st_dev == self->root_device && + stbuf.st_ino == self->root_inode); + } + g_autoptr(GKeyFile) origin = NULL; if (!parse_origin (self, deployment_dfd, deploy_basename, &origin, cancellable, error)) @@ -672,6 +683,8 @@ parse_deployment (OstreeSysroot *self, g_debug ("Deployment %s.%d unlocked=%d", treecsum, deployserial, ret_deployment->unlocked); + if (is_booted_deployment) + self->booted_deployment = g_object_ref (ret_deployment); if (out_deployment) *out_deployment = g_steal_pointer (&ret_deployment); return TRUE; @@ -797,14 +810,30 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, if (!ensure_repo (self, error)) return FALSE; - /* If we didn't check already, see if we have the global ostree-booted flag; + /* Gather some global state; first if we have the global ostree-booted flag; * we'll use it to sanity check that we found a booted deployment for example. + * Second, we also find out whether sysroot == /. */ if (!self->loaded) { if (!glnx_fstatat_allow_noent (AT_FDCWD, "/run/ostree-booted", NULL, 0, error)) return FALSE; self->ostree_booted = (errno == 0); + + { struct stat root_stbuf; + if (!glnx_fstatat (AT_FDCWD, "/", &root_stbuf, 0, error)) + return FALSE; + self->root_device = root_stbuf.st_dev; + self->root_inode = root_stbuf.st_ino; + } + + struct stat self_stbuf; + if (!glnx_fstat (self->sysroot_fd, &self_stbuf, error)) + return FALSE; + + self->root_is_sysroot = + (self->root_device == self_stbuf.st_dev && + self->root_inode == self_stbuf.st_ino); } int bootversion = 0; @@ -847,11 +876,19 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, { OstreeBootconfigParser *config = boot_loader_configs->pdata[i]; + /* Note this also sets self->booted_deployment */ if (!list_deployments_process_one_boot_entry (self, config, deployments, cancellable, error)) - return FALSE; + { + g_clear_object (&self->booted_deployment); + return FALSE; + } } + if (self->ostree_booted && self->root_is_sysroot + && !self->booted_deployment) + return glnx_throw (error, "Unexpected state: /run/ostree-booted found and in / sysroot but not in a booted deployment"); + g_ptr_array_sort (deployments, compare_deployments_by_boot_loader_version_reversed); for (guint i = 0; i < deployments->len; i++) { @@ -859,13 +896,6 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, ostree_deployment_set_index (deployment, i); } - if (!find_booted_deployment (self, deployments, &self->booted_deployment, - cancellable, error)) - return FALSE; - /* Sanity check; note the converse case is fine */ - if (self->booted_deployment && !self->ostree_booted) - return glnx_throw (error, "Unexpected state: In a booted deployment but no /run/ostree-booted?"); - /* Determine whether we're "physical" or not, the first time we initialize */ if (!self->loaded) { @@ -1104,84 +1134,6 @@ _ostree_sysroot_join_lines (GPtrArray *lines) return g_string_free (buf, FALSE); } -static gboolean -parse_kernel_commandline (OstreeKernelArgs **out_args, - GCancellable *cancellable, - GError **error) -{ - g_autoptr(GFile) proc_cmdline = g_file_new_for_path ("/proc/cmdline"); - g_autofree char *contents = NULL; - gsize len; - - if (!g_file_load_contents (proc_cmdline, cancellable, &contents, &len, NULL, - error)) - return FALSE; - - g_strchomp (contents); - *out_args = _ostree_kernel_args_from_string (contents); - return TRUE; -} - -static gboolean -find_booted_deployment (OstreeSysroot *self, - GPtrArray *deployments, - OstreeDeployment **out_deployment, - GCancellable *cancellable, - GError **error) -{ - struct stat root_stbuf; - struct stat self_stbuf; - g_autoptr(OstreeDeployment) ret_deployment = NULL; - - if (stat ("/", &root_stbuf) != 0) - return glnx_throw_errno_prefix (error, "stat /"); - - if (!ensure_sysroot_fd (self, error)) - return FALSE; - - if (fstat (self->sysroot_fd, &self_stbuf) != 0) - return glnx_throw_errno_prefix (error, "fstat"); - - if (root_stbuf.st_dev == self_stbuf.st_dev && - root_stbuf.st_ino == self_stbuf.st_ino) - { - g_autoptr(OstreeKernelArgs) kernel_args = NULL; - if (!parse_kernel_commandline (&kernel_args, cancellable, error)) - return FALSE; - - const char *bootlink_arg = _ostree_kernel_args_get_last_value (kernel_args, "ostree"); - if (bootlink_arg) - { - for (guint i = 0; i < deployments->len; i++) - { - OstreeDeployment *deployment = deployments->pdata[i]; - g_autofree char *deployment_path = ostree_sysroot_get_deployment_dirpath (self, deployment); - struct stat stbuf; - - if (fstatat (self->sysroot_fd, deployment_path, &stbuf, 0) != 0) - return glnx_throw_errno_prefix (error, "fstatat"); - - if (stbuf.st_dev == root_stbuf.st_dev && - stbuf.st_ino == root_stbuf.st_ino) - { - ret_deployment = g_object_ref (deployment); - break; - } - } - - if (ret_deployment == NULL) - return glnx_throw (error, "Unexpected state: ostree= kernel argument found, but / is not a deployment root"); - } - else - { - /* Not an ostree system */ - } - } - - ot_transfer_out_value (out_deployment, &ret_deployment); - return TRUE; -} - /** * ostree_sysroot_query_deployments_for: * @self: Sysroot From 10fb74025d9b9b0fe124918ccbead7965c801d85 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 9 Mar 2018 13:26:07 -0500 Subject: [PATCH 33/51] tests/str: Rework invocation Let's make our `run.sh` generically support any playbook. This is prep for writing further tests in Ansible. Along with that, rework the Ansible so that `tests.yml` is a playbook, and then the other bits are just task lists. It's easier to read. I also started to add a `use_git_build` variable with the idea that we'll be able to run these same tests against an upstream image by setting that variable off. Closes: #1493 Approved by: jlebon --- .papr.yml | 2 +- tests/fedora-str/overlay-git.yml | 56 +++++++++----------- tests/fedora-str/{run.sh => playbook-run.sh} | 7 ++- tests/fedora-str/sysinstall-tests.yml | 37 ++++++------- tests/fedora-str/tests.yml | 12 ++++- 5 files changed, 57 insertions(+), 57 deletions(-) rename tests/fedora-str/{run.sh => playbook-run.sh} (81%) diff --git a/.papr.yml b/.papr.yml index bb84e689..da06fa95 100644 --- a/.papr.yml +++ b/.papr.yml @@ -15,7 +15,7 @@ tests: - ./tests/fedora-str/provision.sh # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/ - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest - - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/run.sh + - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/tests.yml artifacts: - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log diff --git a/tests/fedora-str/overlay-git.yml b/tests/fedora-str/overlay-git.yml index 41d12846..dc0623db 100644 --- a/tests/fedora-str/overlay-git.yml +++ b/tests/fedora-str/overlay-git.yml @@ -1,31 +1,27 @@ ---- -- hosts: localhost - tags: - - atomic - remote_user: root - tasks: - - command: ostree --version - changed_when: False - register: ostree_orig_version - - set_fact: - ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}" - - name: Copy locally built RPMs - synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes - - shell: ostree admin unlock || true - # Install the RPMs we already have. For the test suite we use rpm2cpio - # since it depends on libsoup, but we're not using that yet for the sysinstalled tests - - shell: > - /usr/bin/rpm -Fvh /root/x86_64/*.rpm && \ - cd / && rpm2cpio /root/x86_64/ostree-tests-2*.rpm | cpio -div - - command: ostree --version - register: ostree_new_version - - set_fact: - ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}" - - name: "Fail if we didn't change the ostree version" - when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] - fail: - msg: "Failed to change ostree version" +# Run "admin unlock" and add locally built RPMs, then +# copy the whole tests/ directory into the VM. +- command: ostree --version + changed_when: False + register: ostree_orig_version +- set_fact: + ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}" +- name: Copy locally built RPMs + synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes +- shell: ostree admin unlock || true +# Install the RPMs we already have. For the test suite we use rpm2cpio +# since it depends on libsoup, but we're not using that yet for the sysinstalled tests +- shell: > + /usr/bin/rpm -Fvh /root/x86_64/*.rpm && \ + cd / && rpm2cpio /root/x86_64/ostree-tests-2*.rpm | cpio -div +- command: ostree --version + register: ostree_new_version +- set_fact: + ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}" +- name: "Fail if we didn't change the ostree version" + when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] + fail: + msg: "Failed to change ostree version" - # Next copy all of the tests/ directory - - name: Copy test data - synchronize: src=../../ dest=/root/tests/ archive=yes +# Next copy all of the tests/ directory +- name: Copy test data + synchronize: src=../../ dest=/root/tests/ archive=yes diff --git a/tests/fedora-str/run.sh b/tests/fedora-str/playbook-run.sh similarity index 81% rename from tests/fedora-str/run.sh rename to tests/fedora-str/playbook-run.sh index 8714a85b..1499d085 100755 --- a/tests/fedora-str/run.sh +++ b/tests/fedora-str/playbook-run.sh @@ -1,9 +1,8 @@ #!/usr/bin/bash +# A thin wrapper for ansible-playbook which has a nice check for +# TEST_SUBJECTS being set. set -xeuo pipefail -dn=$(cd $(dirname $0) && pwd) -cd ${dn} - # https://fedoraproject.org/wiki/CI/Tests if test -z "${TEST_SUBJECTS:-}"; then cat < /root/installed-tests.log - register: sysinstalled_result - failed_when: False - - name: Fetch sysinstalled results - fetch: - src: /root/installed-tests.log - dest: artifacts/installed-tests.log - flat: yes - - name: Assert that sysinstalled tests succeeded - when: sysinstalled_result.rc != 0 - fail: - msg: "sysinstalled tests failed" +# Run the system installed tests +- import_tasks: overlay-git.yml + when: use_git_build +# Down the line perhaps do each log file separately? +- name: Run sysinstalled tests + shell: /root/tests/installed/run.sh &> /root/installed-tests.log + register: sysinstalled_result + failed_when: False +- name: Fetch sysinstalled results + fetch: + src: /root/installed-tests.log + dest: artifacts/installed-tests.log + flat: yes +- name: Assert that sysinstalled tests succeeded + when: sysinstalled_result.rc != 0 + fail: + msg: "sysinstalled tests failed" diff --git a/tests/fedora-str/tests.yml b/tests/fedora-str/tests.yml index 7a78adb3..0c60fd07 100644 --- a/tests/fedora-str/tests.yml +++ b/tests/fedora-str/tests.yml @@ -1,2 +1,10 @@ -- include: overlay-git.yml -- include: sysinstall-tests.yml +# This entrypoint right now just runs the sysinstalled-tests. +--- +- hosts: localhost + tags: + - atomic + remote_user: root + vars: + use_git_build: True + tasks: + - import_tasks: sysinstall-tests.yml From 296ef25e12d1c4a49e98b3f9e0a86a31b3762a98 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 16 Mar 2018 14:19:28 -0400 Subject: [PATCH 34/51] lib/core: Support : syntax when listing refs Allow users to pass `:` to list all refs we have locally belonging to ``. Also (re-)allow the similar `:.` syntax for backwards compatibility with flatpak. Closes: #1500 Approved by: cgwalters --- src/libostree/ostree-repo-refs.c | 20 ++++++++++++++++++-- tests/test-refs.sh | 7 +++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index b96cfad0..ad0fd57c 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -629,8 +629,24 @@ _ostree_repo_list_refs_internal (OstreeRepo *self, const char *prefix_path; const char *path; - if (!ostree_parse_refspec (refspec_prefix, &remote, &ref_prefix, error)) - return FALSE; + /* special-case ":" and ":.", which ostree_parse_refspec won't like */ + if (g_str_has_suffix (refspec_prefix, ":") || + g_str_has_suffix (refspec_prefix, ":.")) + { + const char *colon = strrchr (refspec_prefix, ':'); + g_autofree char *r = g_strndup (refspec_prefix, colon - refspec_prefix); + if (ostree_validate_remote_name (r, NULL)) + { + remote = g_steal_pointer (&r); + ref_prefix = g_strdup ("."); + } + } + + if (!ref_prefix) + { + if (!ostree_parse_refspec (refspec_prefix, &remote, &ref_prefix, error)) + return FALSE; + } if (!(flags & OSTREE_REPO_LIST_REFS_EXT_EXCLUDE_REMOTES) && remote) { diff --git a/tests/test-refs.sh b/tests/test-refs.sh index 50862d76..f4fe1833 100755 --- a/tests/test-refs.sh +++ b/tests/test-refs.sh @@ -147,6 +147,13 @@ ${CMD_PREFIX} ostree --repo=repo refs local1 --create=origin:local1 ${CMD_PREFIX} ostree --repo=repo refs | wc -l > refscount.create6 assert_file_has_content refscount.create6 "^11$" +#Check that we can list just remote refs +${CMD_PREFIX} ostree --repo=repo refs origin: | wc -l > refscount.create7 +assert_file_has_content refscount.create7 "^2$" # origin:remote1 origin:local1 +#Also support :. for backcompat with flatpak +${CMD_PREFIX} ostree --repo=repo refs origin:. | wc -l > refscount.create8 +assert_file_has_content refscount.create8 "^2$" # origin:remote1 origin:local1 + echo "ok refs" # Test symlinking a ref From 4e4436beec35f4f48d9750fa39f0a3d7fa94de7e Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 14 Mar 2018 10:36:48 -0400 Subject: [PATCH 35/51] lib/fetcher: Allow clients to append to User-Agent We do already have `http-headers`, which potentially could be used to allow clients to completely override the field, but it seems like the more common use case is simply to append. Closes: #1496 Approved by: cgwalters --- src/libostree/ostree-fetcher-curl.c | 17 ++++++++++++++++- src/libostree/ostree-fetcher-soup.c | 28 ++++++++++++++++++++++++++++ src/libostree/ostree-fetcher.h | 3 +++ src/libostree/ostree-repo-pull.c | 14 ++++++++++++++ src/ostree/ot-builtin-pull.c | 7 +++++++ tests/libtest.sh | 9 ++++----- tests/test-remote-cookies.sh | 3 ++- tests/test-remote-headers.sh | 22 ++++++++++++++++++++-- 8 files changed, 94 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index c4514251..c0f38131 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -77,6 +77,7 @@ struct OstreeFetcher char *proxy; struct curl_slist *extra_headers; int tmpdir_dfd; + char *custom_user_agent; GMainContext *mainctx; CURLM *multi; @@ -180,6 +181,7 @@ _ostree_fetcher_finalize (GObject *object) g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source); if (self->mainctx) g_main_context_unref (self->mainctx); + g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free); G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); } @@ -676,6 +678,18 @@ _ostree_fetcher_set_extra_headers (OstreeFetcher *self, } } +void +_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent) +{ + g_clear_pointer (&self->custom_user_agent, (GDestroyNotify)g_free); + if (extra_user_agent) + { + self->custom_user_agent = + g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent); + } +} + /* Re-bind all of the outstanding curl items to our new main context */ static void adopt_steal_mainctx (OstreeFetcher *self, @@ -716,7 +730,8 @@ initiate_next_curl_request (FetcherRequest *req, curl_easy_setopt (req->easy, CURLOPT_URL, uri); } - curl_easy_setopt (req->easy, CURLOPT_USERAGENT, OSTREE_FETCHER_USERAGENT_STRING); + curl_easy_setopt (req->easy, CURLOPT_USERAGENT, + self->custom_user_agent ?: OSTREE_FETCHER_USERAGENT_STRING); if (self->extra_headers) curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers); diff --git a/src/libostree/ostree-fetcher-soup.c b/src/libostree/ostree-fetcher-soup.c index 306e2534..08a0a700 100644 --- a/src/libostree/ostree-fetcher-soup.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -374,6 +374,24 @@ session_thread_set_tls_database_cb (ThreadClosure *thread_closure, } } +static void +session_thread_set_extra_user_agent_cb (ThreadClosure *thread_closure, + gpointer data) +{ + const char *extra_user_agent = data; + if (extra_user_agent != NULL) + { + g_autofree char *ua = + g_strdup_printf ("%s %s", OSTREE_FETCHER_USERAGENT_STRING, extra_user_agent); + g_object_set (thread_closure->session, SOUP_SESSION_USER_AGENT, ua, NULL); + } + else + { + g_object_set (thread_closure->session, SOUP_SESSION_USER_AGENT, + OSTREE_FETCHER_USERAGENT_STRING, NULL); + } +} + static void on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data); @@ -774,6 +792,16 @@ _ostree_fetcher_set_extra_headers (OstreeFetcher *self, (GDestroyNotify) g_variant_unref); } +void +_ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent) +{ + session_thread_idle_add (self->thread_closure, + session_thread_set_extra_user_agent_cb, + g_strdup (extra_user_agent), + (GDestroyNotify) g_free); +} + static gboolean finish_stream (OstreeFetcherPendingURI *pending, GCancellable *cancellable, diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 7ac82170..32f3ea1b 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -114,6 +114,9 @@ void _ostree_fetcher_set_tls_database (OstreeFetcher *self, void _ostree_fetcher_set_extra_headers (OstreeFetcher *self, GVariant *extra_headers); +void _ostree_fetcher_set_extra_user_agent (OstreeFetcher *self, + const char *extra_user_agent); + guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher *self); void _ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 766098cb..89c67c8e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -87,6 +87,7 @@ typedef struct { OstreeAsyncProgress *progress; GVariant *extra_headers; + char *append_user_agent; gboolean dry_run; gboolean dry_run_emitted_progress; @@ -2922,11 +2923,13 @@ repo_remote_fetch_summary (OstreeRepo *self, const char *url_override = NULL; g_autoptr(GVariant) extra_headers = NULL; g_autoptr(GPtrArray) mirrorlist = NULL; + const char *append_user_agent = NULL; if (options) { (void) g_variant_lookup (options, "override-url", "&s", &url_override); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &extra_headers); + (void) g_variant_lookup (options, "append-user-agent", "&s", &append_user_agent); } mainctx = g_main_context_new (); @@ -2939,6 +2942,9 @@ repo_remote_fetch_summary (OstreeRepo *self, if (extra_headers) _ostree_fetcher_set_extra_headers (fetcher, extra_headers); + if (append_user_agent) + _ostree_fetcher_set_extra_user_agent (fetcher, append_user_agent); + { g_autofree char *url_string = NULL; if (metalink_url_string) @@ -3055,6 +3061,9 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, if (pull_data->extra_headers) _ostree_fetcher_set_extra_headers (pull_data->fetcher, pull_data->extra_headers); + if (pull_data->append_user_agent) + _ostree_fetcher_set_extra_user_agent (pull_data->fetcher, pull_data->append_user_agent); + return TRUE; } @@ -3240,6 +3249,7 @@ initiate_request (OtPullData *pull_data, * * http-headers (a(ss)): Additional headers to add to all HTTP requests * * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid * * localcache-repos (as): File paths for local repos to use as caches when doing remote fetches + * * append-user-agent (s): Additional string to append to the user agent */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3311,6 +3321,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); (void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check); + (void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent); if (pull_data->remote_refspec_name != NULL) pull_data->remote_name = g_strdup (pull_data->remote_refspec_name); @@ -4323,6 +4334,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&pull_data->remote_repo_local); g_free (pull_data->remote_name); + g_free (pull_data->append_user_agent); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref); @@ -5502,6 +5514,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, copy_option (&options_dict, &local_options_dict, "http-headers", G_VARIANT_TYPE ("a(ss)")); copy_option (&options_dict, &local_options_dict, "subdirs", G_VARIANT_TYPE ("as")); copy_option (&options_dict, &local_options_dict, "update-frequency", G_VARIANT_TYPE ("u")); + copy_option (&options_dict, &local_options_dict, "append-user-agent", G_VARIANT_TYPE ("s")); local_options = g_variant_dict_end (&local_options_dict); @@ -5725,6 +5738,7 @@ ostree_repo_resolve_keyring_for_collection (OstreeRepo *self, * * - override-url (s): Fetch summary from this URL if remote specifies no metalink in options * - http-headers (a(ss)): Additional headers to add to all HTTP requests + * - append-user-agent (s): Additional string to append to the user agent * * Returns: %TRUE on success, %FALSE on failure */ diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index c8e12e2d..e2d1f09a 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -41,6 +41,7 @@ static gboolean opt_bareuseronly_files; static char** opt_subpaths; static char** opt_http_headers; static char* opt_cache_dir; +static char* opt_append_user_agent; static int opt_depth = 0; static int opt_frequency = 0; static char* opt_url; @@ -69,6 +70,8 @@ static GOptionEntry options[] = { { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" }, { "timestamp-check", 'T', 0, G_OPTION_ARG_NONE, &opt_timestamp_check, "Require fetched commits to have newer timestamps", NULL }, + /* let's leave this hidden for now; we just need it for tests */ + { "append-user-agent", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &opt_append_user_agent, "Append string to user agent", NULL }, { NULL } }; @@ -333,6 +336,10 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, g_variant_new_variant (g_variant_builder_end (&hdr_builder))); } + if (opt_append_user_agent) + g_variant_builder_add (&builder, "{s@v}", "append-user-agent", + g_variant_new_variant (g_variant_new_string (opt_append_user_agent))); + if (!opt_dry_run) { if (console.is_tty) diff --git a/tests/libtest.sh b/tests/libtest.sh index 9591a88f..586bf9ef 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -234,10 +234,9 @@ ostree_repo_init() { # The original one; use setup_fake_remote_repo2 for newer code, # down the line we'll try to port tests. setup_fake_remote_repo1() { - mode=$1 - commit_opts=${2:-} - args=${3:-} - shift + mode=$1; shift + commit_opts=${1:-} + [ $# -eq 0 ] || shift oldpwd=`pwd` mkdir ostree-srv cd ostree-srv @@ -263,7 +262,7 @@ setup_fake_remote_repo1() { mkdir ${test_tmpdir}/httpd cd httpd ln -s ${test_tmpdir}/ostree-srv ostree - ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args + ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port "$@" port=$(cat ${test_tmpdir}/httpd-port) echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address cd ${oldpwd} diff --git a/tests/test-remote-cookies.sh b/tests/test-remote-cookies.sh index 74e30cb5..e94a70d1 100755 --- a/tests/test-remote-cookies.sh +++ b/tests/test-remote-cookies.sh @@ -27,7 +27,8 @@ echo '1..4' . $(dirname $0)/libtest.sh setup_fake_remote_repo1 "archive" "" \ - "--expected-cookies foo=bar --expected-cookies baz=badger" + --expected-cookies foo=bar \ + --expected-cookies baz=badger assert_fail (){ if $@; then diff --git a/tests/test-remote-headers.sh b/tests/test-remote-headers.sh index 6ba612c0..a4ee386f 100755 --- a/tests/test-remote-headers.sh +++ b/tests/test-remote-headers.sh @@ -25,8 +25,13 @@ echo '1..2' . $(dirname $0)/libtest.sh +V=$($CMD_PREFIX ostree --version | \ + python3 -c 'import sys, yaml; print(yaml.safe_load(sys.stdin)["libostree"]["Version"])') + setup_fake_remote_repo1 "archive" "" \ - "--expected-header foo=bar --expected-header baz=badger" + --expected-header foo=bar \ + --expected-header baz=badger \ + --expected-header "User-Agent=libostree/$V dodo/2.15" assert_fail (){ set +e @@ -46,9 +51,22 @@ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat # Sanity check the setup, without headers the pull should fail assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main +# without proper User-Agent, the pull should fail +assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main \ + --http-header foo=bar \ + --http-header baz=badger +assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main \ + --http-header foo=bar \ + --http-header baz=badger \ + --append-user-agent bar/1.2 + echo "ok setup done" # Now pull should succeed now -${CMD_PREFIX} ostree --repo=repo pull --http-header foo=bar --http-header baz=badger origin main +${CMD_PREFIX} ostree --repo=repo pull \ + --http-header foo=bar \ + --http-header baz=badger \ + --append-user-agent dodo/2.15 \ + origin main echo "ok pull succeeded" From 52f7ba187821fa1bca10b752b3fb28b1fcf8d97a Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Fri, 16 Mar 2018 21:33:11 +0000 Subject: [PATCH 36/51] rust/bupsplit: minor idiomatic fixes This fixes a few un-idiomatic bits in Rust bupsplit code, getting rid of some unchecked casts and an assert statement. Closes: #1502 Approved by: cgwalters --- rust/src/bupsplit.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rust/src/bupsplit.rs b/rust/src/bupsplit.rs index af97e968..607c6639 100644 --- a/rust/src/bupsplit.rs +++ b/rust/src/bupsplit.rs @@ -42,7 +42,7 @@ use std::slice; const ROLLSUM_CHAR_OFFSET: u32 = 31; // Previously in the header file -const BUP_BLOBBITS: u32= 13; +const BUP_BLOBBITS: u32 = 13; const BUP_BLOBSIZE: u32 = (1< () { - let drop_expanded = drop as u32; - let add_expanded = add as u32; + let drop_expanded = u32::from(drop); + let add_expanded = u32::from(add); self.s1 = self.s1.wrapping_add(add_expanded.wrapping_sub(drop_expanded)); self.s2 = self.s2.wrapping_add(self.s1.wrapping_sub(BUP_WINDOWSIZE * (drop_expanded + ROLLSUM_CHAR_OFFSET))); } @@ -102,11 +102,11 @@ pub extern fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, bits: *mut libc::c_int) -> libc::c_int { - let sbuf = unsafe { - assert!(!buf.is_null()); - slice::from_raw_parts(buf, len as usize) - }; + if buf.is_null() { + return 0; + } + let sbuf = unsafe { slice::from_raw_parts(buf, len as usize) }; let mut r = Rollsum::new(); for x in sbuf { r.roll(*x); @@ -115,8 +115,8 @@ pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, let mut sum = r.digest() >> BUP_BLOBBITS; let mut rbits : libc::c_int = BUP_BLOBBITS as i32; while sum & 1 != 0 { - sum = sum >> 1; - rbits = rbits + 1; + sum >>= 1; + rbits += 1; } unsafe { *bits = rbits; From d4d193495f488e71ea89f97f17bd030a6669c61e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 15 Mar 2018 21:15:51 -0400 Subject: [PATCH 37/51] lib/deploy: Port various functions to declare-and-initialize Just noticed this while working on the code. Closes: #1499 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 74 ++++++++++----------------- src/libostree/ostree-sysroot.c | 31 ++++------- 2 files changed, 38 insertions(+), 67 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index f0a667d7..9aa720aa 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -667,27 +667,21 @@ selinux_relabel_dir (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - g_autoptr(GPtrArray) path_parts = g_ptr_array_new (); - g_autoptr(GFileInfo) root_info = NULL; - root_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); + g_autoptr(GFileInfo) root_info = + g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); if (!root_info) - goto out; + return FALSE; + g_autoptr(GPtrArray) path_parts = g_ptr_array_new (); g_ptr_array_add (path_parts, (char*)prefix); if (!relabel_recursively (sysroot, sepolicy, dir, root_info, path_parts, cancellable, error)) - { - g_prefix_error (error, "Relabeling /%s: ", prefix); - goto out; - } + return glnx_prefix_error (error, "Relabeling /%s", prefix); - ret = TRUE; - out: - return ret; + return TRUE; } /* Handles SELinux labeling for /var; this is slated to be deleted. See @@ -766,12 +760,7 @@ merge_configuration (OstreeSysroot *sysroot, if (previous_deployment) { - g_autoptr(GFile) previous_path = NULL; - OstreeBootconfigParser *previous_bootconfig; - - previous_path = ostree_sysroot_get_deployment_directory (sysroot, previous_deployment); - - previous_bootconfig = ostree_deployment_get_bootconfig (previous_deployment); + OstreeBootconfigParser *previous_bootconfig = ostree_deployment_get_bootconfig (previous_deployment); if (previous_bootconfig) { const char *previous_options = ostree_bootconfig_parser_get (previous_bootconfig, "options"); @@ -1939,23 +1928,18 @@ deployment_bootconfigs_equal (OstreeDeployment *a, return FALSE; { - OstreeBootconfigParser *a_bootconfig = ostree_deployment_get_bootconfig (a); - OstreeBootconfigParser *b_bootconfig = ostree_deployment_get_bootconfig (b); - const char *a_boot_options = ostree_bootconfig_parser_get (a_bootconfig, "options"); - const char *b_boot_options = ostree_bootconfig_parser_get (b_bootconfig, "options"); - g_autoptr(OstreeKernelArgs) a_kargs = NULL; - g_autoptr(OstreeKernelArgs) b_kargs = NULL; - g_autofree char *a_boot_options_without_ostree = NULL; - g_autofree char *b_boot_options_without_ostree = NULL; - /* We checksum the kernel arguments *except* ostree= */ - a_kargs = _ostree_kernel_args_from_string (a_boot_options); + OstreeBootconfigParser *a_bootconfig = ostree_deployment_get_bootconfig (a); + const char *a_boot_options = ostree_bootconfig_parser_get (a_bootconfig, "options"); + g_autoptr(OstreeKernelArgs) a_kargs = _ostree_kernel_args_from_string (a_boot_options); _ostree_kernel_args_replace (a_kargs, "ostree"); - a_boot_options_without_ostree = _ostree_kernel_args_to_string (a_kargs); + g_autofree char *a_boot_options_without_ostree = _ostree_kernel_args_to_string (a_kargs); - b_kargs = _ostree_kernel_args_from_string (b_boot_options); + OstreeBootconfigParser *b_bootconfig = ostree_deployment_get_bootconfig (b); + const char *b_boot_options = ostree_bootconfig_parser_get (b_bootconfig, "options"); + g_autoptr(OstreeKernelArgs) b_kargs = _ostree_kernel_args_from_string (b_boot_options); _ostree_kernel_args_replace (b_kargs, "ostree"); - b_boot_options_without_ostree = _ostree_kernel_args_to_string (b_kargs); + g_autofree char *b_boot_options_without_ostree = _ostree_kernel_args_to_string (b_kargs); if (strcmp (a_boot_options_without_ostree, b_boot_options_without_ostree) != 0) return FALSE; @@ -2086,12 +2070,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, GError **error) { gboolean ret = FALSE; - guint i; - gboolean requires_new_bootversion = FALSE; - gboolean found_booted_deployment = FALSE; - gboolean bootloader_is_atomic = FALSE; gboolean boot_was_ro_mount = FALSE; - SyncStats syncstats = { 0, }; g_autoptr(OstreeBootloader) bootloader = NULL; g_assert (self->loaded); @@ -2105,11 +2084,12 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, * matching bootloader configuration, then we can just swap the * subbootversion bootlinks. */ + gboolean requires_new_bootversion = FALSE; if (new_deployments->len != self->deployments->len) requires_new_bootversion = TRUE; else { - for (i = 0; i < new_deployments->len; i++) + for (guint i = 0; i < new_deployments->len; i++) { if (!deployment_bootconfigs_equal (new_deployments->pdata[i], self->deployments->pdata[i])) @@ -2120,7 +2100,8 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, } } - for (i = 0; i < new_deployments->len; i++) + gboolean found_booted_deployment = FALSE; + for (guint i = 0; i < new_deployments->len; i++) { OstreeDeployment *deployment = new_deployments->pdata[i]; g_autoptr(GFile) deployment_root = NULL; @@ -2146,6 +2127,8 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, goto out; } + gboolean bootloader_is_atomic = FALSE; + SyncStats syncstats = { 0, }; if (!requires_new_bootversion) { if (!create_new_bootlinks (self, self->bootversion, @@ -2211,13 +2194,10 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, /* Only show the osname in bootloader titles if there are multiple * osname's among the new deployments. Check for that here. */ - for (i = 1; i < new_deployments->len; i++) + for (guint i = 1; i < new_deployments->len; i++) { - const gchar *osname_0, *osname_i; - - osname_0 = ostree_deployment_get_osname (new_deployments->pdata[0]); - osname_i = ostree_deployment_get_osname (new_deployments->pdata[i]); - + const char *osname_0 = ostree_deployment_get_osname (new_deployments->pdata[0]); + const char *osname_i = ostree_deployment_get_osname (new_deployments->pdata[i]); if (!g_str_equal (osname_0, osname_i)) { show_osname = TRUE; @@ -2225,7 +2205,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, } } - for (i = 0; i < new_deployments->len; i++) + for (guint i = 0; i < new_deployments->len; i++) { OstreeDeployment *deployment = new_deployments->pdata[i]; if (!install_deployment_kernel (self, repo, new_bootversion, diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 1cf7779a..f77d7703 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -690,30 +690,25 @@ parse_deployment (OstreeSysroot *self, return TRUE; } +/* Given a bootloader config, return the value part of the ostree= kernel + * argument. + */ static char * get_ostree_kernel_arg_from_config (OstreeBootconfigParser *config) { - const char *options; - char *ret = NULL; - char **opts, **iter; - - options = ostree_bootconfig_parser_get (config, "options"); + const char *options = ostree_bootconfig_parser_get (config, "options"); if (!options) return NULL; - opts = g_strsplit (options, " ", -1); - for (iter = opts; *iter; iter++) + g_auto(GStrv) opts = g_strsplit (options, " ", -1); + for (char **iter = opts; *iter; iter++) { const char *opt = *iter; if (g_str_has_prefix (opt, "ostree=")) - { - ret = g_strdup (opt + strlen ("ostree=")); - break; - } + return g_strdup (opt + strlen ("ostree=")); } - g_strfreev (opts); - return ret; + return NULL; } static gboolean @@ -963,13 +958,10 @@ ostree_sysroot_get_booted_deployment (OstreeSysroot *self) GPtrArray * ostree_sysroot_get_deployments (OstreeSysroot *self) { - GPtrArray *copy; - guint i; - g_return_val_if_fail (self->loaded, NULL); - copy = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - for (i = 0; i < self->deployments->len; i++) + GPtrArray *copy = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + for (guint i = 0; i < self->deployments->len; i++) g_ptr_array_add (copy, g_object_ref (self->deployments->pdata[i])); return copy; } @@ -1114,10 +1106,9 @@ char * _ostree_sysroot_join_lines (GPtrArray *lines) { GString *buf = g_string_new (""); - guint i; gboolean prev_was_empty = FALSE; - for (i = 0; i < lines->len; i++) + for (guint i = 0; i < lines->len; i++) { const char *line = lines->pdata[i]; /* Special bit to remove extraneous empty lines */ From 01717d7dfc516989f483ad729111793dce934789 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Mar 2018 09:46:20 -0400 Subject: [PATCH 38/51] main, status: Factor out deployment printing into helper Prep for staged deployments; they won't be in the primary deployment list, and we want to print them first. Also pull in some code from rpm-ostree for the red/bold bits and use that tree-wide. Update submodule: libglnx Closes: #1504 Approved by: jlebon --- libglnx | 2 +- src/ostree/main.c | 13 +- src/ostree/ostree-trivial-httpd.c | 13 +- src/ostree/ot-admin-builtin-status.c | 211 +++++++++++++++------------ src/ostree/ot-main.h | 15 ++ 5 files changed, 139 insertions(+), 115 deletions(-) diff --git a/libglnx b/libglnx index 96b1fd95..0c82203c 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 96b1fd9578b3d6ff2d8e0707068f5ef450eba98c +Subproject commit 0c82203cd459a35cc3f471e3205355e9fb79160f diff --git a/src/ostree/main.c b/src/ostree/main.c index ddf531d1..9ed0f880 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -141,15 +141,10 @@ main (int argc, if (error != NULL) { - int is_tty = isatty (1); - const char *prefix = ""; - const char *suffix = ""; - if (is_tty) - { - prefix = "\x1b[31m\x1b[1m"; /* red, bold */ - suffix = "\x1b[22m\x1b[0m"; /* bold off, color reset */ - } - g_printerr ("%serror: %s%s\n", prefix, suffix, error->message); + g_printerr ("%s%serror:%s%s %s\n", + ot_get_red_start (), ot_get_bold_start (), + ot_get_bold_end (), ot_get_red_end (), + error->message); } return ret; diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c index 377de525..5da5a6ab 100644 --- a/src/ostree/ostree-trivial-httpd.c +++ b/src/ostree/ostree-trivial-httpd.c @@ -680,15 +680,10 @@ main (int argc, if (!run (argc, argv, cancellable, &error)) { - int is_tty = isatty (1); - const char *prefix = ""; - const char *suffix = ""; - if (is_tty) - { - prefix = "\x1b[31m\x1b[1m"; /* red, bold */ - suffix = "\x1b[22m\x1b[0m"; /* bold off, color reset */ - } - g_printerr ("%serror: %s%s\n", prefix, suffix, error->message); + g_printerr ("%s%serror:%s%s %s\n", + ot_get_red_start (), ot_get_bold_start (), + ot_get_bold_end (), ot_get_red_end (), + error->message); return 1; } diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index a2e24ef3..096155c6 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -27,6 +27,7 @@ #include "ot-admin-builtins.h" #include "ot-admin-functions.h" #include "ostree.h" +#include "libglnx.h" #include @@ -61,13 +62,116 @@ deployment_get_gpg_verify (OstreeDeployment *deployment, return gpg_verify; } + +static gboolean +deployment_print_status (OstreeSysroot *sysroot, + OstreeRepo *repo, + OstreeDeployment *deployment, + gboolean is_booted, + gboolean is_pending, + gboolean is_rollback, + GCancellable *cancellable, + GError **error) +{ + const char *ref = ostree_deployment_get_csum (deployment); + + /* Load the backing commit; shouldn't normally fail, but if it does, + * we stumble on. + */ + g_autoptr(GVariant) commit = NULL; + (void)ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, ref, + &commit, NULL); + g_autoptr(GVariant) commit_metadata = NULL; + if (commit) + commit_metadata = g_variant_get_child_value (commit, 0); + + const char *version = NULL; + const char *source_title = NULL; + if (commit_metadata) + { + (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_VERSION, "&s", &version); + (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_SOURCE_TITLE, "&s", &source_title); + } + + GKeyFile *origin = ostree_deployment_get_origin (deployment); + + const char *deployment_status = ""; + if (is_pending) + deployment_status = " (pending)"; + else if (is_rollback) + deployment_status = " (rollback)"; + g_print ("%c %s %s.%d%s\n", + is_booted ? '*' : ' ', + ostree_deployment_get_osname (deployment), + ostree_deployment_get_csum (deployment), + ostree_deployment_get_deployserial (deployment), + deployment_status); + if (version) + g_print (" Version: %s\n", version); + + OstreeDeploymentUnlockedState unlocked = ostree_deployment_get_unlocked (deployment); + switch (unlocked) + { + case OSTREE_DEPLOYMENT_UNLOCKED_NONE: + break; + default: + g_print (" %s%sUnlocked: %s%s%s\n", ot_get_red_start (), ot_get_bold_start (), + ostree_deployment_unlocked_state_to_string (unlocked), + ot_get_bold_end (), ot_get_red_end ()); + } + if (ostree_deployment_is_pinned (deployment)) + g_print (" Pinned: yes\n"); + if (!origin) + g_print (" origin: none\n"); + else + { + g_autofree char *origin_refspec = g_key_file_get_string (origin, "origin", "refspec", NULL); + if (!origin_refspec) + g_print (" origin: \n"); + else + g_print (" origin refspec: %s\n", origin_refspec); + if (source_title) + g_print (" `- %s\n", source_title); + } + + if (deployment_get_gpg_verify (deployment, repo)) + { + g_autoptr(GString) output_buffer = g_string_sized_new (256); + /* Print any digital signatures on this commit. */ + + g_autoptr(GError) local_error = NULL; + g_autoptr(OstreeGpgVerifyResult) result = + ostree_repo_verify_commit_ext (repo, ref, NULL, NULL, + cancellable, &local_error); + + /* G_IO_ERROR_NOT_FOUND just means the commit is not signed. */ + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + { + g_clear_error (&local_error); + return TRUE; + } + else if (local_error != NULL) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + + const guint n_signatures = ostree_gpg_verify_result_count_all (result); + for (guint jj = 0; jj < n_signatures; jj++) + { + ostree_gpg_verify_result_describe (result, jj, output_buffer, " GPG: ", + OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT); + } + + g_print ("%s", output_buffer->str); + } + + return TRUE; +} + gboolean ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { - const int is_tty = isatty (1); - const char *red_bold_prefix = is_tty ? "\x1b[31m\x1b[1m" : ""; - const char *red_bold_suffix = is_tty ? "\x1b[22m\x1b[0m" : ""; - g_autoptr(GOptionContext) context = g_option_context_new (""); g_autoptr(OstreeSysroot) sysroot = NULL; @@ -98,98 +202,13 @@ ot_admin_builtin_status (int argc, char **argv, OstreeCommandInvocation *invocat for (guint i = 0; i < deployments->len; i++) { OstreeDeployment *deployment = deployments->pdata[i]; - const char *ref = ostree_deployment_get_csum (deployment); - - /* Load the backing commit; shouldn't normally fail, but if it does, - * we stumble on. - */ - g_autoptr(GVariant) commit = NULL; - (void)ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, ref, - &commit, NULL); - g_autoptr(GVariant) commit_metadata = NULL; - if (commit) - commit_metadata = g_variant_get_child_value (commit, 0); - - const char *version = NULL; - const char *source_title = NULL; - if (commit_metadata) - { - (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_VERSION, "&s", &version); - (void) g_variant_lookup (commit_metadata, OSTREE_COMMIT_META_KEY_SOURCE_TITLE, "&s", &source_title); - } - - GKeyFile *origin = ostree_deployment_get_origin (deployment); - - const char *deployment_status = ""; - if (deployment == pending_deployment) - deployment_status = " (pending)"; - else if (deployment == rollback_deployment) - deployment_status = " (rollback)"; - g_print ("%c %s %s.%d%s\n", - deployment == booted_deployment ? '*' : ' ', - ostree_deployment_get_osname (deployment), - ostree_deployment_get_csum (deployment), - ostree_deployment_get_deployserial (deployment), - deployment_status); - if (version) - g_print (" Version: %s\n", version); - - OstreeDeploymentUnlockedState unlocked = ostree_deployment_get_unlocked (deployment); - switch (unlocked) - { - case OSTREE_DEPLOYMENT_UNLOCKED_NONE: - break; - default: - g_print (" %sUnlocked: %s%s\n", red_bold_prefix, - ostree_deployment_unlocked_state_to_string (unlocked), - red_bold_suffix); - } - if (ostree_deployment_is_pinned (deployment)) - g_print (" Pinned: yes\n"); - if (!origin) - g_print (" origin: none\n"); - else - { - g_autofree char *origin_refspec = g_key_file_get_string (origin, "origin", "refspec", NULL); - if (!origin_refspec) - g_print (" origin: \n"); - else - g_print (" origin refspec: %s\n", origin_refspec); - if (source_title) - g_print (" `- %s\n", source_title); - } - - if (deployment_get_gpg_verify (deployment, repo)) - { - g_autoptr(GString) output_buffer = g_string_sized_new (256); - /* Print any digital signatures on this commit. */ - - g_autoptr(GError) local_error = NULL; - g_autoptr(OstreeGpgVerifyResult) result = - ostree_repo_verify_commit_ext (repo, ref, NULL, NULL, - cancellable, &local_error); - - /* G_IO_ERROR_NOT_FOUND just means the commit is not signed. */ - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) - { - g_clear_error (&local_error); - continue; - } - else if (local_error != NULL) - { - g_propagate_error (error, g_steal_pointer (&local_error)); - return FALSE; - } - - const guint n_signatures = ostree_gpg_verify_result_count_all (result); - for (guint jj = 0; jj < n_signatures; jj++) - { - ostree_gpg_verify_result_describe (result, jj, output_buffer, " GPG: ", - OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT); - } - - g_print ("%s", output_buffer->str); - } + if (!deployment_print_status (sysroot, repo, deployment, + deployment == booted_deployment, + deployment == pending_deployment, + deployment == rollback_deployment, + cancellable, + error)) + return FALSE; } } diff --git a/src/ostree/ot-main.h b/src/ostree/ot-main.h index c5ffeb7f..ac75118c 100644 --- a/src/ostree/ot-main.h +++ b/src/ostree/ot-main.h @@ -91,3 +91,18 @@ gboolean ostree_ensure_repo_writable (OstreeRepo *repo, GError **error); void ostree_print_gpg_verify_result (OstreeGpgVerifyResult *result); gboolean ot_enable_tombstone_commits (OstreeRepo *repo, GError **error); + +/* Copied from rpm-ostree's rpmostree-libbuiltin.h */ +#define TERM_ESCAPE_SEQUENCE(type,seq) \ + static inline const char* ot_get_##type (void) { \ + if (glnx_stdout_is_tty ()) \ + return seq; \ + return ""; \ + } + +TERM_ESCAPE_SEQUENCE(red_start, "\x1b[31m") +TERM_ESCAPE_SEQUENCE(red_end, "\x1b[22m") +TERM_ESCAPE_SEQUENCE(bold_start, "\x1b[1m") +TERM_ESCAPE_SEQUENCE(bold_end, "\x1b[0m") + +#undef TERM_ESCAPE_SEQUENCE From ce2449ad2e4df7c8c5ed1ed1f37a3c038ed9e9c1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Mar 2018 11:57:20 -0400 Subject: [PATCH 39/51] lib/deploy: Use in-function error prefixing more Pulling some of this out of stage deploy work. It's generally better as it's easier to change functions to have multiple callers. Closes: #1505 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 40 ++++++++------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 9aa720aa..87e3b54e 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -756,6 +756,7 @@ merge_configuration (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("During /etc merge", error); g_autoptr(OstreeSePolicy) sepolicy = NULL; if (previous_deployment) @@ -842,6 +843,7 @@ write_origin_file_internal (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Writing out origin file", error); GKeyFile *origin = new_origin ? new_origin : ostree_deployment_get_origin (deployment); @@ -1489,6 +1491,7 @@ create_new_bootlinks (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Creating new current bootlinks", error); glnx_autofd int ostree_dfd = -1; if (!glnx_opendirat (self->sysroot_fd, "ostree", TRUE, &ostree_dfd, error)) return FALSE; @@ -1551,6 +1554,7 @@ swap_bootlinks (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Swapping new version bootlinks", error); glnx_autofd int ostree_dfd = -1; if (!glnx_opendirat (self->sysroot_fd, "ostree", TRUE, &ostree_dfd, error)) return FALSE; @@ -1620,6 +1624,7 @@ install_deployment_kernel (OstreeSysroot *sysroot, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("Installing kernel", error); OstreeBootconfigParser *bootconfig = ostree_deployment_get_bootconfig (deployment); g_autofree char *deployment_dirpath = ostree_sysroot_get_deployment_dirpath (sysroot, deployment); glnx_autofd int deployment_dfd = -1; @@ -2134,10 +2139,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, if (!create_new_bootlinks (self, self->bootversion, new_deployments, cancellable, error)) - { - g_prefix_error (error, "Creating new current bootlinks: "); - goto out; - } + goto out; if (!full_system_sync (self, &syncstats, cancellable, error)) { @@ -2148,10 +2150,7 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, if (!swap_bootlinks (self, self->bootversion, new_deployments, cancellable, error)) - { - g_prefix_error (error, "Swapping current bootlinks: "); - goto out; - } + goto out; bootloader_is_atomic = TRUE; } @@ -2211,26 +2210,17 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot *self, if (!install_deployment_kernel (self, repo, new_bootversion, deployment, new_deployments->len, show_osname, cancellable, error)) - { - g_prefix_error (error, "Installing kernel: "); - goto out; - } + goto out; } /* Create and swap bootlinks for *new* version */ if (!create_new_bootlinks (self, new_bootversion, new_deployments, cancellable, error)) - { - g_prefix_error (error, "Creating new version bootlinks: "); - goto out; - } + goto out; if (!swap_bootlinks (self, new_bootversion, new_deployments, cancellable, error)) - { - g_prefix_error (error, "Swapping new version bootlinks: "); - goto out; - } + goto out; g_debug ("Using bootloader: %s", bootloader ? g_type_name (G_TYPE_FROM_INSTANCE (bootloader)) : "(none)"); @@ -2441,10 +2431,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, deployment_dfd, &sepolicy, cancellable, error)) - { - g_prefix_error (error, "During /etc merge: "); - return FALSE; - } + return FALSE; if (!selinux_relabel_var_if_needed (self, sepolicy, os_deploy_dfd, cancellable, error)) @@ -2470,10 +2457,7 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, if (!write_origin_file_internal (self, new_deployment, NULL, GLNX_FILE_REPLACE_NODATASYNC, cancellable, error)) - { - g_prefix_error (error, "Writing out origin file: "); - return FALSE; - } + return FALSE; } /* After this, install_deployment_kernel() will set the other boot From 9ca3f76cd20fa409f187b93ee13e560ae07c4c59 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 19 Mar 2018 12:10:31 -0400 Subject: [PATCH 40/51] lib/deploy: Have internal origin writing API take sepolicy Ensures it's labeled consistently. Prep for staged deployments which reworks the logic around when the origin file is written. Closes: #1505 Approved by: jlebon --- src/libostree/ostree-sysroot-deploy.c | 53 ++++++++++++++------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 87e3b54e..93a29ed6 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -837,6 +837,7 @@ merge_configuration (OstreeSysroot *sysroot, */ static gboolean write_origin_file_internal (OstreeSysroot *sysroot, + OstreeSePolicy *sepolicy, OstreeDeployment *deployment, GKeyFile *new_origin, GLnxFileReplaceFlags flags, @@ -849,16 +850,21 @@ write_origin_file_internal (OstreeSysroot *sysroot, if (origin) { - g_autofree char *origin_path = NULL; - g_autofree char *contents = NULL; + g_auto(OstreeSepolicyFsCreatecon) con = { 0, }; + if (!_ostree_sepolicy_preparefscreatecon (&con, sepolicy, + "/etc/ostree/remotes.d/dummy.conf", + 0644, error)) + return FALSE; + + g_autofree char *origin_path = + g_strdup_printf ("ostree/deploy/%s/deploy/%s.%d.origin", + ostree_deployment_get_osname (deployment), + ostree_deployment_get_csum (deployment), + ostree_deployment_get_deployserial (deployment)); + + gsize len; - - origin_path = g_strdup_printf ("ostree/deploy/%s/deploy/%s.%d.origin", - ostree_deployment_get_osname (deployment), - ostree_deployment_get_csum (deployment), - ostree_deployment_get_deployserial (deployment)); - - contents = g_key_file_to_data (origin, &len, error); + g_autofree char *contents = g_key_file_to_data (origin, &len, error); if (!contents) return FALSE; @@ -891,7 +897,12 @@ ostree_sysroot_write_origin_file (OstreeSysroot *sysroot, GCancellable *cancellable, GError **error) { - if (!write_origin_file_internal (sysroot, deployment, new_origin, + g_autoptr(GFile) rootfs = g_file_new_for_path ("/"); + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new (rootfs, cancellable, error); + if (!sepolicy) + return FALSE; + + if (!write_origin_file_internal (sysroot, sepolicy, deployment, new_origin, GLNX_FILE_REPLACE_DATASYNC_NEW, cancellable, error)) return FALSE; @@ -2444,21 +2455,13 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, return FALSE; } - { g_auto(OstreeSepolicyFsCreatecon) con = { 0, }; - - if (!_ostree_sepolicy_preparefscreatecon (&con, sepolicy, - "/etc/ostree/remotes.d/dummy.conf", - 0644, error)) - return FALSE; - - /* Don't fsync here, as we assume that's all done in - * ostree_sysroot_write_deployments(). - */ - if (!write_origin_file_internal (self, new_deployment, NULL, - GLNX_FILE_REPLACE_NODATASYNC, - cancellable, error)) - return FALSE; - } + /* Don't fsync here, as we assume that's all done in + * ostree_sysroot_write_deployments(). + */ + if (!write_origin_file_internal (self, sepolicy, new_deployment, NULL, + GLNX_FILE_REPLACE_NODATASYNC, + cancellable, error)) + return FALSE; /* After this, install_deployment_kernel() will set the other boot * options and write it out to disk. From 671b026fcce2f88fbbd2629df2473c52b368e181 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 21 Mar 2018 10:14:12 -0400 Subject: [PATCH 41/51] Release 2018.3 It's been over a month since 2018.2; we have a few features and various fixes, and the "stage" work pending which is pretty invasive. Time for a new release! Closes: #1506 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 7 ++----- src/libostree/libostree-released.sym | 6 ++++++ tests/test-symbols.sh | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 3547f4f3..d903d51e 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ m4_define([year_version], [2018]) m4_define([release_version], [3]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 285ba5f5..bbd1b409 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,11 +18,8 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2018.3 { - ostree_deployment_origin_remove_transient_state; - ostree_sysroot_deployment_set_pinned; - ostree_deployment_is_pinned; -} LIBOSTREE_2018.2; +LIBOSTREE_2018.4 { +} LIBOSTREE_2018.3; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index 6f86ddcd..e9a95cc7 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -458,6 +458,12 @@ LIBOSTREE_2018.2 { ostree_commit_get_content_checksum; } LIBOSTREE_2018.1; +LIBOSTREE_2018.3 { + ostree_deployment_origin_remove_transient_state; + ostree_sysroot_deployment_set_pinned; + ostree_deployment_is_pinned; +} LIBOSTREE_2018.2; + /* NOTE: Only add more content here in release commits! See the * comments at the top of this file. */ diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index ffea2482..13b47afb 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -54,7 +54,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt < Date: Wed, 21 Mar 2018 10:17:01 -0400 Subject: [PATCH 42/51] Post-release version bump Closes: #1506 Approved by: jlebon --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index d903d51e..87070cd8 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2018]) -m4_define([release_version], [3]) +m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From e08b38072f493d769f259868b5b17dc292f20c2e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 21 Mar 2018 16:42:07 -0400 Subject: [PATCH 43/51] ci/papr: Fix artifacts Of course we only notice these things are broken when the CI breaks. Also add some comments and flip flatpak to `required: true` since it should be now. Closes: #1507 Approved by: jlebon --- .papr.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.papr.yml b/.papr.yml index da06fa95..bdb74ec9 100644 --- a/.papr.yml +++ b/.papr.yml @@ -40,7 +40,13 @@ tests: - ci/build-check.sh - ci/ci-release-build.sh +artifacts: + - test-suite.log + - config.log + - gdtr-results + --- +# And now the contexts below here are variant container builds context: f27-rust inherit: true @@ -123,6 +129,7 @@ tests: --- +# Reset inheritance for non-variant builds inherit: false branches: - master @@ -130,7 +137,7 @@ branches: - try context: f27-flatpak -required: false +required: true # This test case wants an "unprivileged container with bubblewrap", # which we don't have right now; so just provision a VM and do a From d48ccbcabfa701a6161c133404812168e0dabc9c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 10:37:07 -0400 Subject: [PATCH 44/51] tests/str: Rework sysinstall-tests to be an entrypoint Prep for creating more types of tests. Move copying of `tests/` into the sysinstall-tests rather than `overlay-git` as not all test types may need that. Factored out of https://github.com/ostreedev/ostree/pull/1501 Closes: #1509 Approved by: jlebon --- .papr.yml | 2 +- tests/fedora-str/overlay-git.yml | 4 --- tests/fedora-str/sysinstall-tests.yml | 45 +++++++++++++++++---------- tests/fedora-str/tests.yml | 10 ------ 4 files changed, 29 insertions(+), 32 deletions(-) delete mode 100644 tests/fedora-str/tests.yml diff --git a/.papr.yml b/.papr.yml index bdb74ec9..501aeecf 100644 --- a/.papr.yml +++ b/.papr.yml @@ -15,7 +15,7 @@ tests: - ./tests/fedora-str/provision.sh # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/ - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest - - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/tests.yml + - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/sysinstall-tests.yml artifacts: - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log diff --git a/tests/fedora-str/overlay-git.yml b/tests/fedora-str/overlay-git.yml index dc0623db..65fe9379 100644 --- a/tests/fedora-str/overlay-git.yml +++ b/tests/fedora-str/overlay-git.yml @@ -21,7 +21,3 @@ when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] fail: msg: "Failed to change ostree version" - -# Next copy all of the tests/ directory -- name: Copy test data - synchronize: src=../../ dest=/root/tests/ archive=yes diff --git a/tests/fedora-str/sysinstall-tests.yml b/tests/fedora-str/sysinstall-tests.yml index 7990e1f7..88b289d4 100644 --- a/tests/fedora-str/sysinstall-tests.yml +++ b/tests/fedora-str/sysinstall-tests.yml @@ -1,17 +1,28 @@ -# Run the system installed tests -- import_tasks: overlay-git.yml - when: use_git_build -# Down the line perhaps do each log file separately? -- name: Run sysinstalled tests - shell: /root/tests/installed/run.sh &> /root/installed-tests.log - register: sysinstalled_result - failed_when: False -- name: Fetch sysinstalled results - fetch: - src: /root/installed-tests.log - dest: artifacts/installed-tests.log - flat: yes -- name: Assert that sysinstalled tests succeeded - when: sysinstalled_result.rc != 0 - fail: - msg: "sysinstalled tests failed" +# This entrypoint right now just runs the sysinstalled-tests. +--- +- hosts: localhost + tags: + - atomic + remote_user: root + vars: + use_git_build: True + tasks: + - import_tasks: overlay-git.yml + when: use_git_build + # Next copy all of the tests/ directory + - name: Copy test data + synchronize: src=../../ dest=/root/tests/ archive=yes + # Down the line perhaps do each log file separately? + - name: Run shell script sysinstalled tests + shell: /root/tests/installed/run.sh &> /root/installed-tests.log + register: sysinstalled_result + failed_when: False + - name: Fetch sysinstalled results + fetch: + src: /root/installed-tests.log + dest: artifacts/installed-tests.log + flat: yes + - name: Assert that sysinstalled tests succeeded + when: sysinstalled_result.rc != 0 + fail: + msg: "sysinstalled tests failed" diff --git a/tests/fedora-str/tests.yml b/tests/fedora-str/tests.yml deleted file mode 100644 index 0c60fd07..00000000 --- a/tests/fedora-str/tests.yml +++ /dev/null @@ -1,10 +0,0 @@ -# This entrypoint right now just runs the sysinstalled-tests. ---- -- hosts: localhost - tags: - - atomic - remote_user: root - vars: - use_git_build: True - tasks: - - import_tasks: sysinstall-tests.yml From dca1cfa60f96619ec84c6dacb9b8f40d2ba7031a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 10:55:16 -0400 Subject: [PATCH 45/51] tests/str: Verify standard-test-roles is installed This tripped me up when regenerating my dev container. Closes: #1509 Approved by: jlebon --- tests/fedora-str/playbook-run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/fedora-str/playbook-run.sh b/tests/fedora-str/playbook-run.sh index 1499d085..96e438d5 100755 --- a/tests/fedora-str/playbook-run.sh +++ b/tests/fedora-str/playbook-run.sh @@ -19,6 +19,9 @@ EOF fi ls -al ${TEST_SUBJECTS} +# This is required +rpm -q standard-test-roles + export ANSIBLE_INVENTORY=${ANSIBLE_INVENTORY:-$(test -e inventory && echo inventory || echo /usr/share/ansible/inventory)} ls -al /dev/kvm exec ansible-playbook --tags=atomic "$@" From bb9cd1eb7213180ceaa1c913b9565f97e08331cc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 11:36:33 -0400 Subject: [PATCH 46/51] tests/installed: Use temporary directories more consistently This is prep for splitting off "nondestructive" tests which we can run in parallel from the destructive/invasive ones which e.g. change the host refspec, do deployments. The `cd` invocation in `prepare_tmpdir` wasn't working because we were running it in a subshell. Fix this by dropping the subshell. Closes: #1509 Approved by: jlebon --- tests/installed/itest-bare-unit.sh | 11 +--- .../installed/itest-bareuser-nouserxattrs.sh | 3 +- tests/installed/itest-payload-link.sh | 53 +++++++++---------- tests/installed/itest-pull-space.sh | 2 +- tests/installed/itest-pull.sh | 4 +- tests/installed/itest-remotes.sh | 2 +- tests/installed/libinsttest.sh | 4 +- 7 files changed, 32 insertions(+), 47 deletions(-) diff --git a/tests/installed/itest-bare-unit.sh b/tests/installed/itest-bare-unit.sh index 28ea4651..fe07f24f 100755 --- a/tests/installed/itest-bare-unit.sh +++ b/tests/installed/itest-bare-unit.sh @@ -15,14 +15,7 @@ dn=$(dirname $0) export G_TEST_SRCDIR=$(realpath $dn/../..) # Use /var/tmp to hopefully use XFS + O_TMPFILE etc. -tempdir=$(mktemp -d /var/tmp/tap-test.XXXXXX) -touch ${tempdir}/.testtmp -function cleanup () { - if test -f ${tempdir}/.testtmp; then - rm "${tempdir}" -rf - fi -} -trap cleanup EXIT -cd ${tempdir} +prepare_tmpdir /var/tmp +trap _tmpdir_cleanup EXIT /usr/libexec/installed-tests/libostree/test-basic.sh /usr/libexec/installed-tests/libostree/test-basic-c diff --git a/tests/installed/itest-bareuser-nouserxattrs.sh b/tests/installed/itest-bareuser-nouserxattrs.sh index 06d3e109..c8c07948 100755 --- a/tests/installed/itest-bareuser-nouserxattrs.sh +++ b/tests/installed/itest-bareuser-nouserxattrs.sh @@ -10,9 +10,8 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libinsttest.sh -test_tmpdir=$(prepare_tmpdir) +prepare_tmpdir trap _tmpdir_cleanup EXIT -cd ${test_tmpdir} mkdir mnt mount -t tmpfs tmpfs mnt diff --git a/tests/installed/itest-payload-link.sh b/tests/installed/itest-payload-link.sh index 807d497d..7469f43e 100755 --- a/tests/installed/itest-payload-link.sh +++ b/tests/installed/itest-payload-link.sh @@ -26,8 +26,9 @@ dn=$(dirname $0) echo "1..1" -cd /var/srv -mkdir repo +# Use /var/tmp so we have O_TMPFILE etc. +prepare_tmpdir /var/tmp +trap _tmpdir_cleanup EXIT ostree --repo=repo init --mode=archive echo -e '[archive]\nzlib-level=1\n' >> repo/config host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,') @@ -39,12 +40,10 @@ run_tmp_webserver $(pwd)/repo origin=$(cat ${test_tmpdir}/httpd-address) cleanup() { - cd ${oldpwd} + cd ${test_tmpdir} umount mnt || true - test -n "${blkdev}" && losetup -d ${blkdev} || true - rm -rf mnt testblk.img + test -n "${blkdev:-}" && losetup -d ${blkdev} || true } -oldpwd=`pwd` trap cleanup EXIT mkdir mnt @@ -57,29 +56,25 @@ fi mkfs.xfs -m reflink=1 ${blkdev} mount ${blkdev} mnt +cd mnt -test_tmpdir=$(pwd)/mnt -cd ${test_tmpdir} - +# Test that coreutils will do reflink touch a -if cp --reflink a b; then - mkdir repo - ostree --repo=repo init - ostree config --repo=repo set core.payload-link-threshold 0 - ostree --repo=repo remote add origin --set=gpg-verify=false ${origin} - ostree --repo=repo pull --disable-static-deltas origin ${host_nonremoteref} - find repo -type l -name '*.payload-link' >payload-links.txt - assert_not_streq "$(wc -l < payload-links.txt)" "0" +cp --reflink a b +mkdir repo +ostree --repo=repo init +ostree config --repo=repo set core.payload-link-threshold 0 +ostree --repo=repo remote add origin --set=gpg-verify=false ${origin} +ostree --repo=repo pull --disable-static-deltas origin ${host_nonremoteref} +find repo -type l -name '*.payload-link' >payload-links.txt +assert_not_streq "$(wc -l < payload-links.txt)" "0" - # Disable logging for inner loop, otherwise it'd be enormous - set +x - cat payload-links.txt | while read i; do - payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) - payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1) - assert_streq "${payload_checksum}" "${payload_checksum_calculated}" - done - set -x - echo "ok pull creates .payload-link" -else - echo "ok # SKIP no reflink support in the file system" -fi +# Disable logging for inner loop, otherwise it'd be enormous +set +x +cat payload-links.txt | while read i; do + payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) + payload_checksum_calculated=$(sha256sum $(readlink -f $i) | cut -d ' ' -f 1) + assert_streq "${payload_checksum}" "${payload_checksum_calculated}" +done +set -x +echo "ok pull creates .payload-link" diff --git a/tests/installed/itest-pull-space.sh b/tests/installed/itest-pull-space.sh index 36703a40..8d218b15 100755 --- a/tests/installed/itest-pull-space.sh +++ b/tests/installed/itest-pull-space.sh @@ -6,7 +6,7 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libinsttest.sh -test_tmpdir=$(prepare_tmpdir) +prepare_tmpdir trap _tmpdir_cleanup EXIT cd ${test_tmpdir} diff --git a/tests/installed/itest-pull.sh b/tests/installed/itest-pull.sh index 65cb9449..b80acdbc 100755 --- a/tests/installed/itest-pull.sh +++ b/tests/installed/itest-pull.sh @@ -7,12 +7,10 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libinsttest.sh -test_tmpdir=$(prepare_tmpdir) +prepare_tmpdir /var/tmp trap _tmpdir_cleanup EXIT # Take the host's ostree, and make it archive -cd /var/srv -rm repo bare-repo -rf mkdir repo ostree --repo=repo init --mode=archive echo -e '[archive]\nzlib-level=1\n' >> repo/config diff --git a/tests/installed/itest-remotes.sh b/tests/installed/itest-remotes.sh index 9b48b68f..8a79015f 100755 --- a/tests/installed/itest-remotes.sh +++ b/tests/installed/itest-remotes.sh @@ -7,7 +7,7 @@ set -xeuo pipefail dn=$(dirname $0) . ${dn}/libinsttest.sh -test_tmpdir=$(prepare_tmpdir) +prepare_tmpdir trap _tmpdir_cleanup EXIT ostree remote list > remotes.txt diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh index d0ee8c6c..6d6ba50c 100644 --- a/tests/installed/libinsttest.sh +++ b/tests/installed/libinsttest.sh @@ -30,10 +30,10 @@ function _tmpdir_cleanup () { fi } prepare_tmpdir() { - test_tmpdir=$(mktemp -d) + local tmpdir=${1:-/tmp} + test_tmpdir=$(mktemp -p ${tmpdir} -d ostree-insttest.XXXXXXXX) touch ${test_tmpdir}/.testtmp cd ${test_tmpdir} - echo ${test_tmpdir} } # This is copied from flatpak/flatpak/tests/test-webserver.sh From d2c7c550c156fbc4d1a8e7edbfd91166c44e6fca Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 12:45:51 -0400 Subject: [PATCH 47/51] tests: Small tweaks for local iteration Support e.g. `-e tests=payload-link`, to choose specific tests for more rapid iteration, and allow skipping tmpdir cleanup to be able to debug. Closes: #1509 Approved by: jlebon --- tests/fedora-str/sysinstall-tests.yml | 3 +++ tests/installed/libinsttest.sh | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/fedora-str/sysinstall-tests.yml b/tests/fedora-str/sysinstall-tests.yml index 88b289d4..58f216ae 100644 --- a/tests/fedora-str/sysinstall-tests.yml +++ b/tests/fedora-str/sysinstall-tests.yml @@ -6,6 +6,7 @@ remote_user: root vars: use_git_build: True + tests: "" tasks: - import_tasks: overlay-git.yml when: use_git_build @@ -15,6 +16,8 @@ # Down the line perhaps do each log file separately? - name: Run shell script sysinstalled tests shell: /root/tests/installed/run.sh &> /root/installed-tests.log + environment: + TESTS: "{{ tests }}" register: sysinstalled_result failed_when: False - name: Fetch sysinstalled results diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh index 6d6ba50c..96da9545 100644 --- a/tests/installed/libinsttest.sh +++ b/tests/installed/libinsttest.sh @@ -25,7 +25,8 @@ dn=$(dirname $0) # Copy of bits from tap-test test_tmpdir= function _tmpdir_cleanup () { - if test -n "${test_tmpdir}" && test -f ${test_tmpdir}/.testtmp; then + if test -z "${TEST_SKIP_CLEANUP:-}" && + test -n "${test_tmpdir}" && test -f ${test_tmpdir}/.testtmp; then rm "${test_tmpdir}" -rf fi } From 246a7a5cc21cefa57a908f4374a7e19766fb89cc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 12:47:07 -0400 Subject: [PATCH 48/51] tests/payload-link: Just test a single duplicate object We were previously assuming that the host content had duplicates, which...hopefully it doesn't! We shouldn't rely on that. Also this test is slow in production and flaky. Let's just test a single duplicate object. Closes: #1509 Approved by: jlebon --- tests/installed/itest-payload-link.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/installed/itest-payload-link.sh b/tests/installed/itest-payload-link.sh index 7469f43e..f0576d82 100755 --- a/tests/installed/itest-payload-link.sh +++ b/tests/installed/itest-payload-link.sh @@ -31,9 +31,16 @@ prepare_tmpdir /var/tmp trap _tmpdir_cleanup EXIT ostree --repo=repo init --mode=archive echo -e '[archive]\nzlib-level=1\n' >> repo/config -host_nonremoteref=$(echo ${host_refspec} | sed 's,[^:]*:,,') -ostree --repo=repo pull-local /ostree/repo ${host_commit} -ostree --repo=repo refs ${host_commit} --create=${host_nonremoteref} + +mkdir content +cd content +dd if=/dev/urandom of=bigobject bs=4k count=2560 +cp --reflink=auto bigobject bigobject2 +# Different metadata, same content +chown bin:bin bigobject2 +cd .. +ostree --repo=repo commit -b dupobjects --consume --selinux-policy=/ --tree=dir=content +ostree --repo=repo summary -u run_tmp_webserver $(pwd)/repo @@ -53,23 +60,24 @@ if ! blkdev=$(losetup --find --show $(pwd)/testblk.img); then exit 0 fi +# This filesystem must support reflinks mkfs.xfs -m reflink=1 ${blkdev} mount ${blkdev} mnt cd mnt -# Test that coreutils will do reflink +# Test that reflink is really there (not just --reflink=auto) touch a cp --reflink a b mkdir repo ostree --repo=repo init ostree config --repo=repo set core.payload-link-threshold 0 ostree --repo=repo remote add origin --set=gpg-verify=false ${origin} -ostree --repo=repo pull --disable-static-deltas origin ${host_nonremoteref} +ostree --repo=repo pull --disable-static-deltas origin dupobjects find repo -type l -name '*.payload-link' >payload-links.txt -assert_not_streq "$(wc -l < payload-links.txt)" "0" +assert_streq "$(wc -l < payload-links.txt)" "1" -# Disable logging for inner loop, otherwise it'd be enormous +# Disable logging for inner loop set +x cat payload-links.txt | while read i; do payload_checksum=$(basename $(dirname $i))$(basename $i .payload-link) From 38cf31f6a72a946a469a8fa649ade2193e041e9e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 10:14:22 -0400 Subject: [PATCH 49/51] switchroot: Ensure /run/ostree-booted is created even without initramfs See https://mail.gnome.org/archives/ostree-list/2018-March/msg00012.html If ostree-prepare-root is run as pid 1 (i.e we're not using an initramfs), then anything we write outside the target sysroot (such as `/run/ostree-booted`) will be lost. Since `ostree-remount.service` runs fairly early in boot, and is triggered via `ConditionKernelCommandLine=ostree`, we can just touch the file there in addition. Closes: #1508 Approved by: akiernan --- src/switchroot/ostree-mount-util.h | 15 +++++++++++++++ src/switchroot/ostree-prepare-root.c | 25 +++++++------------------ src/switchroot/ostree-remount.c | 10 ++++++++++ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/switchroot/ostree-mount-util.h b/src/switchroot/ostree-mount-util.h index 7cf7e09e..08e10f97 100644 --- a/src/switchroot/ostree-mount-util.h +++ b/src/switchroot/ostree-mount-util.h @@ -104,4 +104,19 @@ read_proc_cmdline_ostree (void) return ret; } +/* This is an API for other projects to determine whether or not the + * currently running system is ostree-controlled. + */ +static inline void +touch_run_ostree (void) +{ + int fd = open ("/run/ostree-booted", O_CREAT | O_WRONLY | O_NOCTTY | O_CLOEXEC, 0640); + /* We ignore failures here in case /run isn't mounted...not much we + * can do about that, but we don't want to fail. + */ + if (fd == -1) + return; + (void) close (fd); +} + #endif /* __OSTREE_MOUNT_UTIL_H_ */ diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index 0b04189e..a5c3c785 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -48,23 +48,6 @@ #include "ostree-mount-util.h" -/* This is an API for other projects to determine whether or not the - * currently running system is ostree-controlled. - */ -static void -touch_run_ostree (void) -{ - int fd; - - fd = open ("/run/ostree-booted", O_CREAT | O_WRONLY | O_NOCTTY | O_CLOEXEC, 0640); - /* We ignore failures here in case /run isn't mounted...not much we - * can do about that, but we don't want to fail. - */ - if (fd == -1) - return; - (void) close (fd); -} - static char* resolve_deploy_path (const char * root_mountpoint) { @@ -205,7 +188,13 @@ main(int argc, char *argv[]) err (EXIT_FAILURE, "failed to bind mount (class:readonly) /usr"); } - touch_run_ostree (); + + /* We only stamp /run now if we're running in an initramfs, i.e. we're + * not pid 1. Otherwise it's handled later via ostree-remount.service. + * https://mail.gnome.org/archives/ostree-list/2018-March/msg00012.html + */ + if (getpid () != 1) + touch_run_ostree (); if (strcmp(root_mountpoint, "/") == 0) { diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 3644a063..c3e39c0b 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -46,6 +46,16 @@ main(int argc, char *argv[]) struct stat stbuf; int i; + /* See comments in ostree-prepare-root.c for this. + * + * This service is triggered via + * ConditionKernelCommandLine=ostree + * but it's a lot easier for various bits of userspace to check for + * a file versus parsing the kernel cmdline. So let's ensure + * the stamp file is created here too. + */ + touch_run_ostree (); + /* The /sysroot mount needs to be private to avoid having a mount for e.g. /var/cache * also propagate to /sysroot/ostree/deploy/$stateroot/var/cache * From 83b97ec56901041ac4e9c43da86614f757de13af Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 13:11:07 -0400 Subject: [PATCH 50/51] lib/deltas: Squash some GCC maybe-uninitialized warnings These show up in the RPM build, I didn't yet try to figure out why we're not reproducing them outside of that. Closes: #1510 Approved by: jlebon --- src/libostree/ostree-repo-static-delta-compilation.c | 10 ++-------- src/libostree/ostree-repo-static-delta-core.c | 10 ++++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 746a1a50..9084a72f 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -241,14 +241,8 @@ finish_part (OstreeStaticDeltaBuilder *builder, GError **error) g_variant_builder_add_value (&xattr_builder, part_builder->xattrs->pdata[j]); { - g_autoptr(GBytes) payload_b; - g_autoptr(GBytes) operations_b; - - payload_b = g_string_free_to_bytes (part_builder->payload); - part_builder->payload = NULL; - - operations_b = g_string_free_to_bytes (part_builder->operations); - part_builder->operations = NULL; + g_autoptr(GBytes) payload_b = g_string_free_to_bytes (g_steal_pointer (&part_builder->payload)); + g_autoptr(GBytes) operations_b = g_string_free_to_bytes (g_steal_pointer (&part_builder->operations)); delta_part_content = g_variant_new ("(a(uuu)aa(ayay)@ay@ay)", &mode_builder, &xattr_builder, diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index bca638b2..68b06b5c 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -38,10 +38,8 @@ _ostree_static_delta_parse_checksum_array (GVariant *array, guint *out_n_checksums, GError **error) { - gsize n = g_variant_n_children (array); - guint n_checksums; - - n_checksums = n / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN; + const gsize n = g_variant_n_children (array); + const guint n_checksums = n / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN; if (G_UNLIKELY(n > (G_MAXUINT32/OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) || (n_checksums * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) != n)) @@ -166,8 +164,8 @@ _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, GCancellable *cancellable, GError **error) { - guint8 *checksums_data; - guint n_checksums; + guint8 *checksums_data = NULL; + guint n_checksums = 0; gboolean have_object = TRUE; if (!_ostree_static_delta_parse_checksum_array (checksum_array, From 155c936cd57892adab7b0b4ed4473da7fc45caa3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 22 Mar 2018 16:52:04 -0400 Subject: [PATCH 51/51] Release 2018.4 A quick turnaround to include one PR: https://github.com/ostreedev/ostree/pull/1508 "switchroot: Ensure /run/ostree-booted is created even without initramfs" This fixes ostree when booting without an initramfs. Thanks to @akiernan for the bug report and helping review the fix! I'm working on enhancing the test suite, which will help in adding some coverage here. Also for this release I'm going to avoid adding a "stub" symbol section to the `-released.sym` file; I don't believe it's necessary. Closes: #1512 Approved by: jlebon --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 87070cd8..35962dfa 100644 --- a/configure.ac +++ b/configure.ac @@ -7,7 +7,7 @@ m4_define([year_version], [2018]) m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index bbd1b409..3377ae12 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,7 +18,7 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2018.4 { +LIBOSTREE_2018.5 { } LIBOSTREE_2018.3; /* Stub section for the stable release *after* this development one; don't