From 8f2c240326defc2315ba53115b7784d7f2b1ad8f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 13 Mar 2020 21:56:14 -0400 Subject: [PATCH 001/177] Post-release version bump --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index f8d7b2be..e2d867e7 100644 --- a/configure.ac +++ b/configure.ac @@ -2,15 +2,15 @@ AC_PREREQ([2.63]) dnl To do a release: follow the instructions to update libostree-released.sym from dnl libostree-devel.sym, update the checksum in test-symbols.sh, set is_release_build=yes dnl below. Then make another post-release commit to bump the version and set -dnl is_release_build=yes +dnl is_release_build=no. dnl Seed the release notes with `git-shortlog-with-prs ..`. Then use dnl `git-evtag` to create the tag and push it. Finally, create a GitHub release and attach dnl the tarball from `make dist`. m4_define([year_version], [2020]) -m4_define([release_version], [3]) +m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) From 0b6ac72b7ff4beba18d578d4c3668cc385351026 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Mar 2020 02:13:15 +0000 Subject: [PATCH 002/177] main/pin: Fix usage of GError This regressed in https://github.com/ostreedev/ostree/commit/2db79fb398fba2f9ab2e05c517ebccfe1834674b I noticed this while finally getting the installed tests to run in FCOS via kola and `ostree admin pin 0` is now aborting because we were returning TRUE, but no error set. I don't see a reason to try to continue on if we hit an error; the original reporter was requesting support for multiple arguments, but not "ignore invalid requests". --- src/ostree/ot-admin-builtin-pin.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/ostree/ot-admin-builtin-pin.c b/src/ostree/ot-admin-builtin-pin.c index ced0fcf3..d4337e33 100644 --- a/src/ostree/ot-admin-builtin-pin.c +++ b/src/ostree/ot-admin-builtin-pin.c @@ -52,39 +52,28 @@ ot_admin_builtin_pin (int argc, char **argv, OstreeCommandInvocation *invocation return FALSE; } - unsigned int nsuccess = 0; for (unsigned int i = 1; i < argc; i++) { const char *deploy_index_str = argv[i]; const int deploy_index = atoi (deploy_index_str); - g_autoptr(GError) e = NULL; - g_autoptr(OstreeDeployment) target_deployment = ot_admin_get_indexed_deployment (sysroot, deploy_index, &e); + g_autoptr(OstreeDeployment) target_deployment = ot_admin_get_indexed_deployment (sysroot, deploy_index, error); if (!target_deployment) - { - g_print ("Invalid deployment %s: %s\n", deploy_index_str, e->message); - continue; - } + return FALSE; gboolean current_pin = ostree_deployment_is_pinned (target_deployment); const gboolean desired_pin = !opt_unpin; if (current_pin == desired_pin) { g_print ("Deployment %s is already %s\n", deploy_index_str, current_pin ? "pinned" : "unpinned"); - nsuccess++; } else { - g_autoptr(GError) e = NULL; - if (ostree_sysroot_deployment_set_pinned (sysroot, target_deployment, desired_pin, &e)) - { - g_print ("Deployment %s is now %s\n", deploy_index_str, desired_pin ? "pinned" : "unpinned"); - nsuccess++; - } - else - g_print ("Failed to %s deployment %s: %s\n", desired_pin ? "pin" : "unpin", deploy_index_str, e->message); + if (!ostree_sysroot_deployment_set_pinned (sysroot, target_deployment, desired_pin, error)) + return FALSE; + g_print ("Deployment %s is now %s\n", deploy_index_str, desired_pin ? "pinned" : "unpinned"); } } - return nsuccess > 0; + return TRUE; } From f7a8617c0151392391e028de91c00d910f461806 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 18 Mar 2020 10:32:52 -0400 Subject: [PATCH 003/177] bin/diff: Clarify documentation around REV and DIR syntax Related: #2032 --- man/ostree-diff.xml | 4 ++-- src/ostree/ot-builtin-diff.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/man/ostree-diff.xml b/man/ostree-diff.xml index 10658cf2..0b7ac892 100644 --- a/man/ostree-diff.xml +++ b/man/ostree-diff.xml @@ -51,7 +51,7 @@ Boston, MA 02111-1307, USA. - ostree diff OPTIONS REV TARGETDIR + ostree diff OPTIONS REV_OR_DIR REV_OR_DIR @@ -59,7 +59,7 @@ Boston, MA 02111-1307, USA. Description - Compare directory TARGETDIR against revision REV. Shows files and directories modified, added, and deleted. If there is a file in TARGETDIR not in REV, it will show with an "A" for "added". If a file in REV is not in TARGETDIR, it shows "D" for "deleted". "M" for "modified" will also show. + Compare a directory or revision against another directory or revision. If REV_OR_DIR starts with `/` or `./`, it is interpreted as a directory, otherwise a revision. Shows files and directories modified, added, and deleted. If there is a file in the second REV_OR_DIR not in the first, it will show with an "A" for "added". If a file in the first REV_OR_DIR is not in the second, it shows "D" for "deleted". "M" for "modified" will also show. diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c index 1f0488de..82a533d5 100644 --- a/src/ostree/ot-builtin-diff.c +++ b/src/ostree/ot-builtin-diff.c @@ -142,7 +142,7 @@ ostree_builtin_diff (int argc, char **argv, OstreeCommandInvocation *invocation, g_autoptr(GPtrArray) removed = NULL; g_autoptr(GPtrArray) added = NULL; - context = g_option_context_new ("REV TARGETDIR"); + context = g_option_context_new ("REV_OR_DIR REV_OR_DIR"); if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error)) goto out; From fd822a8dfdcd2d9a6a9c749144e5fdb37df59b18 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 19 Mar 2020 10:05:57 -0400 Subject: [PATCH 004/177] lib/pull: Don't leave commits pulled by depth as partial When pulling full parent commits via e.g. `--depth N`, we weren't unmarking them as partial in the out path. Closes: #2035 --- src/libostree/ostree-repo-pull.c | 7 +++++++ tests/test-local-pull-depth.sh | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 125c113d..381cce47 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4719,6 +4719,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error)) goto out; } + + /* and finally any parent commits we might also have pulled because of depth>0 */ + GLNX_HASH_TABLE_FOREACH (pull_data->commit_to_depth, const char*, commit) + { + if (!ostree_repo_mark_commit_partial (pull_data->repo, commit, FALSE, error)) + goto out; + } } ret = TRUE; diff --git a/tests/test-local-pull-depth.sh b/tests/test-local-pull-depth.sh index 7080e121..96b20b9c 100755 --- a/tests/test-local-pull-depth.sh +++ b/tests/test-local-pull-depth.sh @@ -34,21 +34,31 @@ ostree_repo_init repo2 --mode="archive" ${CMD_PREFIX} ostree --repo=repo2 pull-local repo find repo2/objects -name '*.commit' | wc -l > commitcount assert_file_has_content commitcount "^1$" +find repo2/state -name '*.commitpartial' | wc -l > commitpartialcount +assert_file_has_content commitpartialcount "^0$" ${CMD_PREFIX} ostree --repo=repo2 pull-local --depth=0 repo find repo2/objects -name '*.commit' | wc -l > commitcount assert_file_has_content commitcount "^1$" +find repo2/state -name '*.commitpartial' | wc -l > commitpartialcount +assert_file_has_content commitpartialcount "^0$" + +${CMD_PREFIX} ostree --repo=repo2 pull-local --depth=1 --commit-metadata-only repo +find repo2/objects -name '*.commit' | wc -l > commitcount +assert_file_has_content commitcount "^2$" +find repo2/state -name '*.commitpartial' | wc -l > commitpartialcount +assert_file_has_content commitpartialcount "^1$" ${CMD_PREFIX} ostree --repo=repo2 pull-local --depth=1 repo find repo2/objects -name '*.commit' | wc -l > commitcount assert_file_has_content commitcount "^2$" - -${CMD_PREFIX} ostree --repo=repo2 pull-local --depth=1 repo -find repo2/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^2$" +find repo2/state -name '*.commitpartial' | wc -l > commitpartialcount +assert_file_has_content commitpartialcount "^0$" ${CMD_PREFIX} ostree --repo=repo2 pull-local --depth=-1 repo find repo2/objects -name '*.commit' | wc -l > commitcount assert_file_has_content commitcount "^2$" +find repo2/state -name '*.commitpartial' | wc -l > commitpartialcount +assert_file_has_content commitpartialcount "^0$" echo "ok local pull depth" From b93180a4d3d95f557826fe613a9ffa16012ebb4a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 11 Mar 2020 20:00:14 +0000 Subject: [PATCH 005/177] =?UTF-8?q?tests:=20Rework=20tests/installed=20?= =?UTF-8?q?=E2=86=92=20tests/kola?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we made an effort to use the [Fedora Standard Test Interface](https://docs.fedoraproject.org/en-US/ci/standard-test-interface/). This effort was not very successful; the primary thing that it really died on is Ansible just didn't support rebooting very well. I think that's since gotten better, but even then, Ansible wasn't the best thing for a test framework for us anyways. In the meantime Fedora CoreOS happened emphasizing Ignition and not "post-hoc reconciliation" models like Ansible over ssh. And, [coreos-assembler](https://github.com/coreos/coreos-assembler) happened too. Furthermore, we really need to test OSTree's interaction with Ignition as we've invented several special things there. Then most recently, I've been working on having cosa/kola support running externally defined tests: https://github.com/coreos/coreos-assembler/pull/1215 There's a lot of things to clean up after this but at least this works for me: ``` $ cd /srv/fcos $ cosa kola run -- --parallel 4 --output-dir tmp/kola -E ~/src/github/ostreedev/ostree/ 'ext.ostree.*' ``` NOTE: This *does not* drop ostree binaries into the target. See: https://github.com/coreos/coreos-assembler/pull/1252#issuecomment-600623315 This drops our dependency on Python in the installed tests, and also fixes a few bugs that came up. I disabled the `itest-bare-user-root.sh` one because it's entangled with the shell script infrastructure for the unit tests. --- tests/installed/README.md | 15 -- tests/installed/destructive-ansible.yml | 16 -- tests/installed/destructive-unit.yml | 42 ----- tests/installed/destructive/README.md | 5 - tests/installed/destructive/var-mount.yml | 13 -- tests/installed/execute_batch.yml | 23 --- tests/installed/libinsttest.sh | 101 ------------ tests/installed/nondestructive.yml | 41 ----- .../installed/nondestructive/libtest-core.sh | 1 - tests/installed/playbook-run.sh | 41 ----- tests/installed/provision.sh | 8 - tests/installed/run.sh | 12 -- .../tasks/disable-all-rpmmd-repos.yml | 5 - tests/installed/tasks/install-git.yml | 23 --- tests/installed/tasks/overlay-git.yml | 23 --- tests/installed/tasks/query-host.yml | 6 - tests/installed/tasks/reboot.yml | 72 -------- tests/kola/README.md | 1 + tests/kola/destructive/data | 1 + .../destructive/itest-bare-root.sh | 5 +- .../destructive/itest-deploy-selinux.sh | 9 +- .../destructive/itest-label-selinux.sh | 3 +- .../destructive/staged-deploy.sh} | 91 ++++++----- tests/kola/destructive/var-mount.sh | 23 +++ tests/kola/libinsttest.sh | 91 +++++++++++ tests/kola/libtest-core.sh | 153 +++++++++++++++++ tests/kola/nondestructive/data | 1 + .../nondestructive/itest-bare-unit.sh | 5 +- .../nondestructive/itest-bare-user-root.sh | 3 +- .../itest-bareuser-nouserxattrs.sh | 3 +- .../nondestructive/itest-payload-link.sh | 3 +- .../nondestructive/itest-pull-space.sh | 10 +- .../nondestructive/itest-pull.sh | 3 +- .../nondestructive/itest-remotes.sh | 3 +- .../nondestructive}/libtest-core.sh | 0 tests/libtest-core.sh | 154 +----------------- 36 files changed, 336 insertions(+), 673 deletions(-) delete mode 100644 tests/installed/README.md delete mode 100644 tests/installed/destructive-ansible.yml delete mode 100644 tests/installed/destructive-unit.yml delete mode 100644 tests/installed/destructive/README.md delete mode 100644 tests/installed/destructive/var-mount.yml delete mode 100644 tests/installed/execute_batch.yml delete mode 100644 tests/installed/libinsttest.sh delete mode 100644 tests/installed/nondestructive.yml delete mode 120000 tests/installed/nondestructive/libtest-core.sh delete mode 100755 tests/installed/playbook-run.sh delete mode 100755 tests/installed/provision.sh delete mode 100755 tests/installed/run.sh delete mode 100644 tests/installed/tasks/disable-all-rpmmd-repos.yml delete mode 100644 tests/installed/tasks/install-git.yml delete mode 100644 tests/installed/tasks/overlay-git.yml delete mode 100644 tests/installed/tasks/query-host.yml delete mode 100644 tests/installed/tasks/reboot.yml create mode 100644 tests/kola/README.md create mode 120000 tests/kola/destructive/data rename tests/{installed => kola}/destructive/itest-bare-root.sh (95%) rename tests/{installed => kola}/destructive/itest-deploy-selinux.sh (91%) rename tests/{installed => kola}/destructive/itest-label-selinux.sh (98%) rename tests/{installed/destructive/staged-deploy.yml => kola/destructive/staged-deploy.sh} (65%) mode change 100644 => 100755 create mode 100755 tests/kola/destructive/var-mount.sh create mode 100644 tests/kola/libinsttest.sh create mode 100644 tests/kola/libtest-core.sh create mode 120000 tests/kola/nondestructive/data rename tests/{installed => kola}/nondestructive/itest-bare-unit.sh (92%) mode change 100755 => 100644 rename tests/{installed => kola}/nondestructive/itest-bare-user-root.sh (97%) rename tests/{installed => kola}/nondestructive/itest-bareuser-nouserxattrs.sh (92%) rename tests/{installed => kola}/nondestructive/itest-payload-link.sh (99%) rename tests/{installed => kola}/nondestructive/itest-pull-space.sh (92%) rename tests/{installed => kola}/nondestructive/itest-pull.sh (98%) rename tests/{installed => kola}/nondestructive/itest-remotes.sh (85%) rename tests/{installed => kola/nondestructive}/libtest-core.sh (100%) mode change 100644 => 120000 tests/libtest-core.sh diff --git a/tests/installed/README.md b/tests/installed/README.md deleted file mode 100644 index b3d14a74..00000000 --- a/tests/installed/README.md +++ /dev/null @@ -1,15 +0,0 @@ -This directory holds tests that use the -[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface). - -The high level structure is that we take a qcow2 file, inject -built RPMs into it, and then use Ansible to run tests. - -See `.papr.yml` for canonical usage. - -For local development, you should cache the qcow2 somewhere -stable (outside of this git repo). Also note that `../ci/build-rpms.sh` -does *not* pick up uncommitted changes! Stated more strongly, you -currently need to run `build-rpms.sh` after every change. - -To run just a specific test, use e.g.: -`env TEST_SUBJECTS=/path/to/qcow2 ./playbook-run.sh -e tests=.*pull nondestructive.yml` diff --git a/tests/installed/destructive-ansible.yml b/tests/installed/destructive-ansible.yml deleted file mode 100644 index 50c8fd77..00000000 --- a/tests/installed/destructive-ansible.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Ansible-based tests. ---- -- hosts: localhost - tags: - - atomic - remote_user: root - vars: - use_git_build: True - tests: "." - tasks: - - import_tasks: tasks/disable-all-rpmmd-repos.yml - - import_tasks: tasks/install-git.yml - when: use_git_build - - import_tasks: tasks/query-host.yml - - import_tasks: destructive/staged-deploy.yml - - import_tasks: destructive/var-mount.yml diff --git a/tests/installed/destructive-unit.yml b/tests/installed/destructive-unit.yml deleted file mode 100644 index e70561d4..00000000 --- a/tests/installed/destructive-unit.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This entrypoint right now just runs shell-script based tests -# from destructive/. Note that we `rpm-ostree usroverlay` git -# builds. So it's not supported to reboot in these tests. -# These tests will be run serially, and can e.g. change deployments. ---- -- hosts: localhost - tags: - - atomic - remote_user: root - vars: - use_git_build: True - tests: "." - tasks: - - import_tasks: tasks/query-host.yml - - set_fact: - rpmostree_initial_deployment: "{{ rpmostree_status[\"deployments\"][0] }}" - - import_tasks: tasks/overlay-git.yml - when: use_git_build - # Next copy all of the tests/ directory - - name: Copy test data - synchronize: src=../../ dest=/root/tests/ archive=yes - - - find: - paths: /root/tests/installed/destructive - patterns: "itest-*.sh" - register: all_tests - - set_fact: - selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}" - - assert: - that: - - "{{ selected_tests|length }} != 0" - - file: path=/root/logs state=directory - - block: - - name: Run destructive tests - shell: "{{ item }} &> /root/logs/$(basename {{ item }}).log" - with_items: - - "{{ selected_tests }}" - always: - - synchronize: - src: /root/logs/ - dest: artifacts/installed-destructive - mode: pull diff --git a/tests/installed/destructive/README.md b/tests/installed/destructive/README.md deleted file mode 100644 index cafe605a..00000000 --- a/tests/installed/destructive/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This suite of tests is run from PAPR. Everything in here is destructive; it's -recommended to only run them in disposable virtual machines. This is done -in `tests/fedora-str/sysinstalled-tests.yml`, which currently uses a single VM -and runs the tests serially. It's likely in the future this will be changed -to do one VM per test. diff --git a/tests/installed/destructive/var-mount.yml b/tests/installed/destructive/var-mount.yml deleted file mode 100644 index 3fe041af..00000000 --- a/tests/installed/destructive/var-mount.yml +++ /dev/null @@ -1,13 +0,0 @@ -# https://github.com/ostreedev/ostree/issues/1667 -- name: Set up /var as a mountpoint - shell: | - set -xeuo pipefail - cp -a /var /sysroot/myvar - touch /sysroot/myvar/somenewfile - echo '/sysroot/myvar /var none bind 0 0' >> /etc/fstab -- include_tasks: ../tasks/reboot.yml -- name: Check that /var mountpoint worked - shell: | - set -xeuo pipefail - systemctl status var.mount - test -f /var/somenewfile diff --git a/tests/installed/execute_batch.yml b/tests/installed/execute_batch.yml deleted file mode 100644 index e4a11bb5..00000000 --- a/tests/installed/execute_batch.yml +++ /dev/null @@ -1,23 +0,0 @@ -##################### -# execute_batch.yml -##################### -- name: Begin async command execution - shell: "{{ async_item }} &> {{ logdir }}/{{ async_item|basename }}.log" - # 10 minutes; the PAPR tester generally times out before that - async: 600 - poll: 0 - with_items: "{{ async_commands }}" - loop_control: - loop_var: "async_item" - register: async_results - -- name: Check async command status - async_status: - jid: "{{ async_result_item.ansible_job_id }}" - with_items: "{{ async_results.results }}" - loop_control: - loop_var: "async_result_item" - register: async_poll_results - until: async_poll_results.finished - retries: 500 - retry_pause: 5 diff --git a/tests/installed/libinsttest.sh b/tests/installed/libinsttest.sh deleted file mode 100644 index edf358f2..00000000 --- a/tests/installed/libinsttest.sh +++ /dev/null @@ -1,101 +0,0 @@ -# Common definitions for installed, privileged tests -# -# Copyright (C) 2017 Colin Walters -# -# SPDX-License-Identifier: LGPL-2.0+ -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -dn=$(dirname $0) -. ${dn}/../libtest-core.sh - -# Copy of bits from tap-test -test_tmpdir= -function _tmpdir_cleanup () { - if test -z "${TEST_SKIP_CLEANUP:-}" && - test -n "${test_tmpdir}" && test -f ${test_tmpdir}/.testtmp; then - rm "${test_tmpdir}" -rf - fi -} -prepare_tmpdir() { - local tmpdir=${1:-/tmp} - test_tmpdir=$(mktemp -p ${tmpdir} -d ostree-insttest.XXXXXXXX) - touch ${test_tmpdir}/.testtmp - cd ${test_tmpdir} -} - -if test -x /usr/bin/python3; then - export PYTHON=/usr/bin/python3 - export PYTHONHTTPSERVER=http.server -elif test -x /usr/bin/python; then - export PYTHON=/usr/bin/python - export PYTHONHTTPSERVER=SimpleHTTPServer -else - fatal "no python found" -fi - -# This is copied from flatpak/flatpak/tests/test-webserver.sh -run_tmp_webserver() { - dir=$1 - - test -n ${test_tmpdir} - - cd ${dir} - env PYTHONUNBUFFERED=1 setsid $PYTHON -m $PYTHONHTTPSERVER 0 &>${test_tmpdir}/httpd-output & - cd - - child_pid=$! - - for x in $(seq 60); do - echo "Waiting for web server ($x/60)..." >&2 - # Snapshot the output - cp ${test_tmpdir}/httpd-output{,.tmp} - # If it's non-empty, see whether it matches our regexp - if test -s ${test_tmpdir}/httpd-output.tmp; then # py3's http.server prints the http:// address also - sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\)\( (http://[^)]*)\)\? \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port - if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then - # If so, we've successfully extracted the port - break - fi - fi - sleep 1 - done - - if [ ! -f ${test_tmpdir}/httpd-port ]; then - cat ${test_tmpdir}/httpd-output - fatal "can't start up httpd" - fi - - port=$(cat ${test_tmpdir}/httpd-port) - echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address - echo "$child_pid" > ${test_tmpdir}/httpd-pid -} - -# Determine our origin refspec - we'll use this as a test base -rpmostree=$(which rpm-ostree 2>/dev/null) -if test -z "${rpmostree}"; then - skip "no rpm-ostree, at some point point this to raw ostree too" -fi - -# We need to be root -assert_streq $(id -u) 0 - -rpmostree_query_json() { - query=$1 - rpm-ostree status --json | $PYTHON -c 'import json,sys; v=json.load(sys.stdin); print(v'${query}')' -} -host_refspec=$(rpmostree_query_json '["deployments"][0]["origin"]') -host_commit=$(rpmostree_query_json '["deployments"][0]["checksum"]') -host_osname=$(rpmostree_query_json '["deployments"][0]["osname"]') diff --git a/tests/installed/nondestructive.yml b/tests/installed/nondestructive.yml deleted file mode 100644 index dcd0d442..00000000 --- a/tests/installed/nondestructive.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Nondestructive sysinstalled tests, run in parallel. ---- -- hosts: localhost - tags: - - atomic - remote_user: root - vars: - use_git_build: True - tests: "." - # Arbitrary...we want some parallelism - batching_factor: 4 - tasks: - - import_tasks: tasks/query-host.yml - - import_tasks: tasks/overlay-git.yml - when: use_git_build - # Next copy all of the tests/ directory - - name: Copy test data - synchronize: src=../../ dest=/root/tests/ archive=yes - - find: - paths: /root/tests/installed/nondestructive - patterns: "itest-*.sh" - register: all_tests - - set_fact: - selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}" - - assert: - that: - - "{{ selected_tests|length }} != 0" - - file: path=/root/logs state=directory - - block: - - name: Run nondestructive tests - vars: - logdir: /root/logs - async_commands: "{{ item }}" - include_tasks: execute_batch.yml - with_items: - - "{{ selected_tests | batch('{{ batching_factor }}') | list }}" - always: - - synchronize: - src: /root/logs - dest: artifacts/installed-nondestructive - mode: pull diff --git a/tests/installed/nondestructive/libtest-core.sh b/tests/installed/nondestructive/libtest-core.sh deleted file mode 120000 index d26203e2..00000000 --- a/tests/installed/nondestructive/libtest-core.sh +++ /dev/null @@ -1 +0,0 @@ -../libtest-core.sh \ No newline at end of file diff --git a/tests/installed/playbook-run.sh b/tests/installed/playbook-run.sh deleted file mode 100755 index af906c3d..00000000 --- a/tests/installed/playbook-run.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/bash -# A thin wrapper for ansible-playbook which has a nice check for -# TEST_SUBJECTS being set. -set -xeuo pipefail - -dn=$(cd $(dirname $0) && pwd) -if ! test -d build; then - mkdir -p build - (cd build && ${dn}/../../ci/build-rpm.sh) -else - # XXX: we should invalidate based on `git describe` or something - echo "NOTE: Re-using prebuilt RPMs:" - find build -name '*.rpm' -fi - -# https://fedoraproject.org/wiki/CI/Tests -if test -z "${TEST_SUBJECTS:-}"; then - cat < - /usr/bin/rpm -Fvh /root/x86_64/*.rpm && \ - cd / && rpm2cpio /root/x86_64/ostree-tests-2*.rpm | cpio -div -- command: ostree --version - register: ostree_new_version -- set_fact: - ostree_new_version_yaml: "{{ ostree_new_version.stdout | from_yaml }}" -- name: "Fail if we didn't change the ostree version" - when: ostree_orig_version_yaml['libostree']['Git'] == ostree_new_version_yaml['libostree']['Git'] - fail: - msg: "Failed to change ostree version" diff --git a/tests/installed/tasks/query-host.yml b/tests/installed/tasks/query-host.yml deleted file mode 100644 index d572ae64..00000000 --- a/tests/installed/tasks/query-host.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: Load status json - command: rpm-ostree status --json - changed_when: False - register: rpmostree_status_json -- set_fact: - rpmostree_status: "{{ rpmostree_status_json.stdout | from_json }}" diff --git a/tests/installed/tasks/reboot.yml b/tests/installed/tasks/reboot.yml deleted file mode 100644 index a5d0cbec..00000000 --- a/tests/installed/tasks/reboot.yml +++ /dev/null @@ -1,72 +0,0 @@ -# This file is copied from atomic-host-tests - -# vim: set ft=ansible: -# There is no clean way to restart hosts in ansible. The general issue is that -# the shutdown command may close sshd before ansible has time to "return" from -# the task, even with async & poll. This is due to the fact that asynchronous -# tasks still require a small synchronous bootstrapping script which takes 1 sec -# to complete, during which it is vulnerable to erroring out if sshd dies. -# To mitigate this, we prefix a sleep command before the shutdown so -# ansible has time to move on. For more info on this issue, see: -# https://github.com/ansible/ansible/issues/10616 -# -# The Ansible docs now recommend this combination of tasks to handle reboots -# https://support.ansible.com/hc/en-us/articles/201958037-Reboot-a-server-and-wait-for-it-to-come-back - -# remember the real ansible_host for following local actions -# (otherwise ansible will target the localhost) -- set_fact: - real_ansible_host: "{{ ansible_host }}" - timeout: "{{ cli_reboot_timeout | default('120') }}" - -# Have to account for both because Fedora STR uses the old version of these -# inventory values for some reason. -- when: ansible_port is defined - set_fact: - real_ansible_port: "{{ ansible_port }}" - -- when: ansible_ssh_port is defined - set_fact: - real_ansible_port: "{{ ansible_ssh_port }}" - -- name: Get original bootid - command: cat /proc/sys/kernel/random/boot_id - register: orig_bootid - -# Stop sshd (thus preventing new connections) and kill our current user's -# connection so that we can't race to get back in to the system while it's -# shutting down -- name: restart hosts - when: (not skip_shutdown is defined) or (not skip_shutdown) - shell: | - systemctl stop sshd - systemd-run --on-active=5 systemctl reboot - async: 1 - poll: 0 - ignore_errors: true - -# NB: The wait_for is executed locally and doesn't require privs, so avoid sudo -- debug: - msg: "Waiting for reboot: {{ ansible_date_time.iso8601 }}" -- wait_for_connection: - delay: 5 - timeout: 120 - search_regex: "OpenSSH" -- debug: - msg: "SSH port is up {{ ansible_date_time.iso8601 }}" - -- name: Assert that the bootid changed - command: cat /proc/sys/kernel/random/boot_id - register: new_bootid - until: new_bootid.stdout != orig_bootid.stdout -- assert: - that: - - new_bootid.stdout != orig_bootid.stdout - -# provide an empty iterator when a list is not provided -# http://docs.ansible.com/ansible/playbooks_conditionals.html#loops-and-conditionals -- name: check services have started - service: - name: "{{ item }}" - state: started - with_items: "{{ wait_for_services|default([]) }}" diff --git a/tests/kola/README.md b/tests/kola/README.md new file mode 100644 index 00000000..19cd5528 --- /dev/null +++ b/tests/kola/README.md @@ -0,0 +1 @@ +This uses https://github.com/coreos/coreos-assembler/pull/1252 diff --git a/tests/kola/destructive/data b/tests/kola/destructive/data new file mode 120000 index 00000000..a96aa0ea --- /dev/null +++ b/tests/kola/destructive/data @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/tests/installed/destructive/itest-bare-root.sh b/tests/kola/destructive/itest-bare-root.sh similarity index 95% rename from tests/installed/destructive/itest-bare-root.sh rename to tests/kola/destructive/itest-bare-root.sh index 055d873c..2834a829 100755 --- a/tests/installed/destructive/itest-bare-root.sh +++ b/tests/kola/destructive/itest-bare-root.sh @@ -4,12 +4,13 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh echo "1..2" date +require_writable_sysroot + cd /ostree/repo/tmp rm co -rf rm co-testref -rf diff --git a/tests/installed/destructive/itest-deploy-selinux.sh b/tests/kola/destructive/itest-deploy-selinux.sh similarity index 91% rename from tests/installed/destructive/itest-deploy-selinux.sh rename to tests/kola/destructive/itest-deploy-selinux.sh index 92f6993c..099b5c27 100755 --- a/tests/installed/destructive/itest-deploy-selinux.sh +++ b/tests/kola/destructive/itest-deploy-selinux.sh @@ -4,8 +4,9 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh + +require_writable_sysroot date # Create a new deployment @@ -35,8 +36,8 @@ ostree admin undeploy 0 cd /ostree/repo/tmp ostree checkout --fsync=0 -H ${host_commit} test-label -rm test-label/usr/lib/ostree-boot/vmlinuz* -rm test-label/usr/lib/ostree-boot/initramfs* +rm -vf test-label/usr/lib/ostree-boot/vmlinuz* +rm -vf test-label/usr/lib/ostree-boot/initramfs* cd test-label/usr/lib/modules/* rm initramfs.img echo new initramfs > initramfs.img diff --git a/tests/installed/destructive/itest-label-selinux.sh b/tests/kola/destructive/itest-label-selinux.sh similarity index 98% rename from tests/installed/destructive/itest-label-selinux.sh rename to tests/kola/destructive/itest-label-selinux.sh index 2a492858..87fb26f0 100755 --- a/tests/installed/destructive/itest-label-selinux.sh +++ b/tests/kola/destructive/itest-label-selinux.sh @@ -4,8 +4,7 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh date cd /ostree/repo/tmp diff --git a/tests/installed/destructive/staged-deploy.yml b/tests/kola/destructive/staged-deploy.sh old mode 100644 new mode 100755 similarity index 65% rename from tests/installed/destructive/staged-deploy.yml rename to tests/kola/destructive/staged-deploy.sh index cfd9165b..47d7c9af --- a/tests/installed/destructive/staged-deploy.yml +++ b/tests/kola/destructive/staged-deploy.sh @@ -1,14 +1,21 @@ -# Test the deploy --stage functionality; first, we stage a deployment -# reboot, and validate that it worked. +#!/bin/bash +set -xeuo pipefail -# for now, until the preset propagates down -- name: Start up path unit - shell: | - set -xeuo pipefail +. ${KOLA_EXT_DATA}/libinsttest.sh + +require_writable_sysroot +prepare_tmpdir + +n=$(nth_boot) +case "${n}" in + 1) + commit=${host_commit} + # Test the deploy --stage functionality; first, we stage a deployment + # reboot, and validate that it worked. + # for now, until the preset propagates down + # Start up path unit systemctl enable --now ostree-finalize-staged.path -- name: Write staged-deploy commit - shell: | - set -xeuo pipefail + # Write staged-deploy commit export OSTREE_SYSROOT_DEBUG="test-staged-path" cd /ostree/repo/tmp # https://github.com/ostreedev/ostree/issues/1569 @@ -20,10 +27,7 @@ systemctl show -p ActiveState ostree-finalize-staged.service | grep -q inactive systemctl show -p TriggeredBy ostree-finalize-staged.service | grep -q path ostree admin deploy --stage staged-deploy | tee out.txt - if ! grep -q 'test-staged-path: Not running' out.txt; then - cat out.txt - exit 1 - fi + assert_file_has_content out.txt 'test-staged-path: Not running' systemctl show -p SubState ostree-finalize-staged.path | grep running systemctl show -p ActiveState ostree-finalize-staged.service | grep active new_mtime=$(stat -c '%.Y' /sysroot/ostree/deploy) @@ -37,24 +41,22 @@ test -f deployment-ref-found rm deployment-ref-found if ostree admin pin 0 2>err.txt; then - echo "Pinned staged deployment"; exit 1 + fatal "Pinned staged deployment" fi - grep -qFe 'Cannot pin staged deployment' err.txt - environment: - commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" -- include_tasks: ../tasks/reboot.yml -- name: Check that deploy-staged service worked - shell: | - set -xeuo pipefail + assert_file_has_content err.txt 'Cannot pin staged deployment' + kola_reboot + ;; + 2) + # Check that deploy-staged service worked rpm-ostree status # Assert that the previous boot had a journal entry for it - journalctl -b "-1" -u ostree-finalize-staged.service | grep -q -e 'Transaction complete' + journalctl -b "-1" -u ostree-finalize-staged.service > svc.txt + assert_file_has_content svc.txt 'Bootloader updated; bootconfig swap: yes; deployment count change: 1' + rm -f svc.txt # And there should not be a staged deployment test '!' -f /run/ostree/staged-deployment -- name: Upgrade with staging - shell: | - set -xeuo pipefail + # Upgrade with staging test '!' -f /run/ostree/staged-deployment ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment @@ -67,27 +69,25 @@ test -f /run/ostree/staged-deployment # Debating bouncing back out to Ansible for this firstdeploycommit=$(rpm-ostree status |grep 'Commit:' |head -1|sed -e 's,^ *Commit: *,,') - test "${firstdeploycommit}" = "${newcommit}" + assert_streq "${firstdeploycommit}" "${newcommit}" # Cleanup rpm-ostree cleanup -rp -- import_tasks: ../tasks/query-host.yml + echo "ok upgrade with staging" -# Ensure we can unstage -- name: Write staged-deploy commit, then unstage - shell: | - set -xeuo pipefail + # Ensure we can unstage + # Write staged-deploy commit, then unstage ostree admin deploy --stage staged-deploy ostree admin status > status.txt - grep -qFe '(staged)' status.txt + assert_file_has_content_literal status.txt '(staged)' test -f /run/ostree/staged-deployment ostree admin undeploy 0 ostree admin status > status.txt grep -vqFe '(staged)' status.txt test '!' -f /run/ostree/staged-deployment + echo "ok unstage" -- name: Staged should be overwritten by non-staged as first - shell: | - set -xeuo pipefail + # Staged should be overwritten by non-staged as first + commit=$(rpmostree_query_json '.deployments[0].checksum') ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment ostree --repo=/ostree/repo refs --create nonstaged-deploy "${commit}" @@ -96,22 +96,23 @@ grep -vqFe '(staged)' status.txt test '!' -f /run/ostree/staged-deployment ostree admin undeploy 0 - environment: - commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" + echo "ok staged overwritten by non-staged" -- name: Staged is retained when pushing rollback - shell: | - set -xeuo pipefail + # Staged is retained when pushing rollback + commit=$(rpmostree_query_json '.deployments[0].checksum') ostree admin deploy --stage staged-deploy test -f /run/ostree/staged-deployment ostree admin deploy --retain-pending --not-as-default nonstaged-deploy test -f /run/ostree/staged-deployment ostree admin status > status.txt - grep -qFe '(staged)' status.txt + assert_file_has_content_literal status.txt '(staged)' ostree admin undeploy 0 ostree admin undeploy 1 - environment: - commit: "{{ rpmostree_status['deployments'][0]['checksum'] }}" + echo "ok staged retained" -- name: Cleanup refs - shell: ostree refs --delete staged-deploy nonstaged-deploy + # Cleanup refs + ostree refs --delete staged-deploy nonstaged-deploy + echo "ok cleanup refs" + ;; + *) fatal "Unexpected boot count" ;; +esac diff --git a/tests/kola/destructive/var-mount.sh b/tests/kola/destructive/var-mount.sh new file mode 100755 index 00000000..b3fe3d98 --- /dev/null +++ b/tests/kola/destructive/var-mount.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# https://github.com/ostreedev/ostree/issues/1667 +set -xeuo pipefail + +. ${KOLA_EXT_DATA}/libinsttest.sh + +n=$(nth_boot) +case "${n}" in + 1) + require_writable_sysroot + # Hack this off for now + chattr -i /sysroot + cp -a /var /sysroot/myvar + touch /sysroot/myvar/somenewfile + echo '/sysroot/myvar /var none bind 0 0' >> /etc/fstab + kola_reboot + ;; + 2) + systemctl status var.mount + test -f /var/somenewfile + ;; + *) fatal "Unexpected boot count $n" +esac diff --git a/tests/kola/libinsttest.sh b/tests/kola/libinsttest.sh new file mode 100644 index 00000000..2552cf78 --- /dev/null +++ b/tests/kola/libinsttest.sh @@ -0,0 +1,91 @@ +# Common definitions for installed, privileged tests +# +# Copyright (C) 2017 Colin Walters +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +. ${KOLA_EXT_DATA}/libtest-core.sh + +# Copy of bits from tap-test +test_tmpdir= +function _tmpdir_cleanup () { + if test -z "${TEST_SKIP_CLEANUP:-}" && + test -n "${test_tmpdir}" && test -f ${test_tmpdir}/.testtmp; then + rm "${test_tmpdir}" -rf + fi +} +prepare_tmpdir() { + local tmpdir=${1:-/var/tmp} + test_tmpdir=$(mktemp -p ${tmpdir} -d ostree-insttest.XXXXXXXX) + touch ${test_tmpdir}/.testtmp + cd ${test_tmpdir} +} + +# This is copied from flatpak/flatpak/tests/test-webserver.sh +run_tmp_webserver() { + dir=$1 + + port=8000 + podman create --name ostree-httpd --privileged -ti --net=host -v "${dir}":/srv --workdir /srv \ + registry.svc.ci.openshift.org/coreos/fedora:31 python3 -m http.server "${port}" + podman generate systemd ostree-httpd > /etc/systemd/system/ostree-httpd.service + systemctl daemon-reload + systemctl start ostree-httpd.service + + address="http://127.0.0.1:${port}" + while ! curl --head "${address}" &>/dev/null; do sleep 1; done + echo "${address}" > ${test_tmpdir}/httpd-address +} + +# Yeah this is a hack. Doing this better requires more first class support +# for creating derived commits in ostree potentially. Or barring that, +# just recommend to people to use `unshare -m` or equivalent and +# mount -o remount,rw /sysroot in their code. +require_writable_sysroot() { + if ! test -w /sysroot; then + mount -o remount,rw /sysroot + fi +} + +nth_boot() { + journalctl --list-boots | wc -l +} + +kola_reboot() { + kill -TERM $$ + sleep 2m + echo "failed to reboot?" 1>&2 + exit 1 +} + +# Determine our origin refspec - we'll use this as a test base +rpmostree=$(which rpm-ostree 2>/dev/null) +if test -z "${rpmostree}"; then + skip "no rpm-ostree, at some point point this to raw ostree too" +fi + +# We need to be root +assert_streq $(id -u) 0 + +rpmostree_query_json() { + query=$1 + rpm-ostree status --json | jq -r "${query}" +} +host_refspec=$(rpmostree_query_json '.deployments[0].origin') +host_commit=$(rpmostree_query_json '.deployments[0].checksum') +host_osname=$(rpmostree_query_json '.deployments[0].osname') diff --git a/tests/kola/libtest-core.sh b/tests/kola/libtest-core.sh new file mode 100644 index 00000000..945d2857 --- /dev/null +++ b/tests/kola/libtest-core.sh @@ -0,0 +1,153 @@ +# Core source library for shell script tests; the +# canonical version lives in: +# +# https://github.com/ostreedev/ostree +# +# Known copies are in the following repos: +# +# - https://github.com/projectatomic/rpm-ostree +# +# Copyright (C) 2017 Colin Walters +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +fatal() { + echo $@ 1>&2; exit 1 +} +# fatal() is shorter to type, but retain this alias +assert_not_reached () { + fatal "$@" +} + +# Some tests look for specific English strings. Use a UTF-8 version +# of the C (POSIX) locale if we have one, or fall back to en_US.UTF-8 +# (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) +# +# If we can't find the locale command assume we have support for C.UTF-8 +# (e.g. musl based systems) +if type -p locale >/dev/null; then + export LC_ALL=$(locale -a | grep -iEe '^(C|en_US)\.(UTF-8|utf8)$' | head -n1 || true) + if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi +else + export LC_ALL=C.UTF-8 +fi +# A GNU extension, used whenever LC_ALL is not C +unset LANGUAGE + +# This should really be the default IMO +export G_DEBUG=fatal-warnings + +assert_streq () { + test "$1" = "$2" || fatal "$1 != $2" +} + +assert_str_match () { + if ! echo "$1" | grep -E -q "$2"; then + fatal "$1 does not match regexp $2" + fi +} + +assert_not_streq () { + (! test "$1" = "$2") || fatal "$1 == $2" +} + +assert_has_file () { + test -f "$1" || fatal "Couldn't find '$1'" +} + +assert_has_dir () { + test -d "$1" || fatal "Couldn't find '$1'" +} + +# Dump ls -al + file contents to stderr, then fatal() +_fatal_print_file() { + file="$1" + shift + ls -al "$file" >&2 + sed -e 's/^/# /' < "$file" >&2 + fatal "$@" +} + +assert_not_has_file () { + if test -f "$1"; then + _fatal_print_file "$1" "File '$1' exists" + fi +} + +assert_not_file_has_content () { + fpath=$1 + shift + for re in "$@"; do + if grep -q -e "$re" "$fpath"; then + _fatal_print_file "$fpath" "File '$fpath' matches regexp '$re'" + fi + done +} + +assert_not_has_dir () { + if test -d "$1"; then + fatal "Directory '$1' exists" + fi +} + +assert_file_has_content () { + fpath=$1 + shift + for re in "$@"; do + if ! grep -q -e "$re" "$fpath"; then + _fatal_print_file "$fpath" "File '$fpath' doesn't match regexp '$re'" + fi + done +} + +assert_file_has_content_literal () { + fpath=$1; shift + for s in "$@"; do + if ! grep -q -F -e "$s" "$fpath"; then + _fatal_print_file "$fpath" "File '$fpath' doesn't match fixed string list '$s'" + fi + done +} + +assert_file_has_mode () { + mode=$(stat -c '%a' $1) + if [ "$mode" != "$2" ]; then + fatal "File '$1' has wrong mode: expected $2, but got $mode" + fi +} + +assert_symlink_has_content () { + if ! test -L "$1"; then + fatal "File '$1' is not a symbolic link" + fi + if ! readlink "$1" | grep -q -e "$2"; then + _fatal_print_file "$1" "Symbolic link '$1' doesn't match regexp '$2'" + fi +} + +assert_file_empty() { + if test -s "$1"; then + _fatal_print_file "$1" "File '$1' is not empty" + fi +} + +# Use to skip all of these tests +skip() { + echo "1..0 # SKIP" "$@" + exit 0 +} diff --git a/tests/kola/nondestructive/data b/tests/kola/nondestructive/data new file mode 120000 index 00000000..a96aa0ea --- /dev/null +++ b/tests/kola/nondestructive/data @@ -0,0 +1 @@ +.. \ No newline at end of file diff --git a/tests/installed/nondestructive/itest-bare-unit.sh b/tests/kola/nondestructive/itest-bare-unit.sh old mode 100755 new mode 100644 similarity index 92% rename from tests/installed/nondestructive/itest-bare-unit.sh rename to tests/kola/nondestructive/itest-bare-unit.sh index c973ee7c..d399d78d --- a/tests/installed/nondestructive/itest-bare-unit.sh +++ b/tests/kola/nondestructive/itest-bare-unit.sh @@ -5,8 +5,9 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh + +fatal "FIXME - need to also sync over the installed tests" date # These tests sort of bypass the installed-tests spec; diff --git a/tests/installed/nondestructive/itest-bare-user-root.sh b/tests/kola/nondestructive/itest-bare-user-root.sh similarity index 97% rename from tests/installed/nondestructive/itest-bare-user-root.sh rename to tests/kola/nondestructive/itest-bare-user-root.sh index 9e04f7bc..ad3d2dac 100755 --- a/tests/installed/nondestructive/itest-bare-user-root.sh +++ b/tests/kola/nondestructive/itest-bare-user-root.sh @@ -4,8 +4,7 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh echo "1..1" date diff --git a/tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh b/tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh similarity index 92% rename from tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh rename to tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh index 7063286b..7908d31b 100755 --- a/tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh +++ b/tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh @@ -7,8 +7,7 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh prepare_tmpdir trap _tmpdir_cleanup EXIT diff --git a/tests/installed/nondestructive/itest-payload-link.sh b/tests/kola/nondestructive/itest-payload-link.sh similarity index 99% rename from tests/installed/nondestructive/itest-payload-link.sh rename to tests/kola/nondestructive/itest-payload-link.sh index 6a6a01d3..6cfe291a 100755 --- a/tests/installed/nondestructive/itest-payload-link.sh +++ b/tests/kola/nondestructive/itest-payload-link.sh @@ -21,8 +21,7 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh echo "1..1" date diff --git a/tests/installed/nondestructive/itest-pull-space.sh b/tests/kola/nondestructive/itest-pull-space.sh similarity index 92% rename from tests/installed/nondestructive/itest-pull-space.sh rename to tests/kola/nondestructive/itest-pull-space.sh index e365b86d..97524f67 100755 --- a/tests/installed/nondestructive/itest-pull-space.sh +++ b/tests/kola/nondestructive/itest-pull-space.sh @@ -3,14 +3,10 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh date prepare_tmpdir -trap _tmpdir_cleanup EXIT - -cd ${test_tmpdir} truncate -s 20MB testblk.img blkdev=$(losetup --find --show $(pwd)/testblk.img) mkfs.xfs ${blkdev} @@ -51,9 +47,7 @@ rm -rf mnt/repo ostree --repo=mnt/repo init --mode=bare-user echo 'fsync=false' >> mnt/repo/config echo 'min-free-space-size=10MB' >> mnt/repo/config -if ostree --repo=mnt/repo pull-local --commit-metadata-only /ostree/repo ${host_commit} 2>err.txt; then - fatal "could not write metadata objects even when min-free-space value should allow it" -fi +ostree --repo=mnt/repo pull-local --commit-metadata-only /ostree/repo ${host_commit} echo "ok metadata write even when free space is lower than min-free-space value" rm -rf mnt/repo diff --git a/tests/installed/nondestructive/itest-pull.sh b/tests/kola/nondestructive/itest-pull.sh similarity index 98% rename from tests/installed/nondestructive/itest-pull.sh rename to tests/kola/nondestructive/itest-pull.sh index 07056ea1..770f2444 100755 --- a/tests/installed/nondestructive/itest-pull.sh +++ b/tests/kola/nondestructive/itest-pull.sh @@ -7,8 +7,7 @@ set -xeuo pipefail # FIXME: https://github.com/ostreedev/ostree/pull/1548 exit 0 -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh date prepare_tmpdir /var/tmp diff --git a/tests/installed/nondestructive/itest-remotes.sh b/tests/kola/nondestructive/itest-remotes.sh similarity index 85% rename from tests/installed/nondestructive/itest-remotes.sh rename to tests/kola/nondestructive/itest-remotes.sh index 836e35ad..d1fb455b 100755 --- a/tests/installed/nondestructive/itest-remotes.sh +++ b/tests/kola/nondestructive/itest-remotes.sh @@ -4,8 +4,7 @@ set -xeuo pipefail -dn=$(dirname $0) -. ${dn}/../libinsttest.sh +. ${KOLA_EXT_DATA}/libinsttest.sh date prepare_tmpdir diff --git a/tests/installed/libtest-core.sh b/tests/kola/nondestructive/libtest-core.sh similarity index 100% rename from tests/installed/libtest-core.sh rename to tests/kola/nondestructive/libtest-core.sh diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh deleted file mode 100644 index 945d2857..00000000 --- a/tests/libtest-core.sh +++ /dev/null @@ -1,153 +0,0 @@ -# Core source library for shell script tests; the -# canonical version lives in: -# -# https://github.com/ostreedev/ostree -# -# Known copies are in the following repos: -# -# - https://github.com/projectatomic/rpm-ostree -# -# Copyright (C) 2017 Colin Walters -# -# SPDX-License-Identifier: LGPL-2.0+ -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -fatal() { - echo $@ 1>&2; exit 1 -} -# fatal() is shorter to type, but retain this alias -assert_not_reached () { - fatal "$@" -} - -# Some tests look for specific English strings. Use a UTF-8 version -# of the C (POSIX) locale if we have one, or fall back to en_US.UTF-8 -# (https://sourceware.org/glibc/wiki/Proposals/C.UTF-8) -# -# If we can't find the locale command assume we have support for C.UTF-8 -# (e.g. musl based systems) -if type -p locale >/dev/null; then - export LC_ALL=$(locale -a | grep -iEe '^(C|en_US)\.(UTF-8|utf8)$' | head -n1 || true) - if [ -z "${LC_ALL}" ]; then fatal "Can't find suitable UTF-8 locale"; fi -else - export LC_ALL=C.UTF-8 -fi -# A GNU extension, used whenever LC_ALL is not C -unset LANGUAGE - -# This should really be the default IMO -export G_DEBUG=fatal-warnings - -assert_streq () { - test "$1" = "$2" || fatal "$1 != $2" -} - -assert_str_match () { - if ! echo "$1" | grep -E -q "$2"; then - fatal "$1 does not match regexp $2" - fi -} - -assert_not_streq () { - (! test "$1" = "$2") || fatal "$1 == $2" -} - -assert_has_file () { - test -f "$1" || fatal "Couldn't find '$1'" -} - -assert_has_dir () { - test -d "$1" || fatal "Couldn't find '$1'" -} - -# Dump ls -al + file contents to stderr, then fatal() -_fatal_print_file() { - file="$1" - shift - ls -al "$file" >&2 - sed -e 's/^/# /' < "$file" >&2 - fatal "$@" -} - -assert_not_has_file () { - if test -f "$1"; then - _fatal_print_file "$1" "File '$1' exists" - fi -} - -assert_not_file_has_content () { - fpath=$1 - shift - for re in "$@"; do - if grep -q -e "$re" "$fpath"; then - _fatal_print_file "$fpath" "File '$fpath' matches regexp '$re'" - fi - done -} - -assert_not_has_dir () { - if test -d "$1"; then - fatal "Directory '$1' exists" - fi -} - -assert_file_has_content () { - fpath=$1 - shift - for re in "$@"; do - if ! grep -q -e "$re" "$fpath"; then - _fatal_print_file "$fpath" "File '$fpath' doesn't match regexp '$re'" - fi - done -} - -assert_file_has_content_literal () { - fpath=$1; shift - for s in "$@"; do - if ! grep -q -F -e "$s" "$fpath"; then - _fatal_print_file "$fpath" "File '$fpath' doesn't match fixed string list '$s'" - fi - done -} - -assert_file_has_mode () { - mode=$(stat -c '%a' $1) - if [ "$mode" != "$2" ]; then - fatal "File '$1' has wrong mode: expected $2, but got $mode" - fi -} - -assert_symlink_has_content () { - if ! test -L "$1"; then - fatal "File '$1' is not a symbolic link" - fi - if ! readlink "$1" | grep -q -e "$2"; then - _fatal_print_file "$1" "Symbolic link '$1' doesn't match regexp '$2'" - fi -} - -assert_file_empty() { - if test -s "$1"; then - _fatal_print_file "$1" "File '$1' is not empty" - fi -} - -# Use to skip all of these tests -skip() { - echo "1..0 # SKIP" "$@" - exit 0 -} diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh new file mode 120000 index 00000000..11131fca --- /dev/null +++ b/tests/libtest-core.sh @@ -0,0 +1 @@ +kola/libtest-core.sh \ No newline at end of file From 53d1efbaab03c089a8ae5fe52cd7a313595b6ff4 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Thu, 19 Mar 2020 15:28:08 -0700 Subject: [PATCH 006/177] lib/fetcher-util: retry download on G_IO_ERROR_PARTIAL_INPUT Add G_IO_ERROR_PARTIAL_INPUT to the list of error codes caused by transient networking errors which lead us to retry the request. When attempting to install the spotify flatpak you often get the error message "Connection terminated unexpectedly" and the download of the deb file fails. In this case, libsoup is setting G_IO_ERROR_PARTIAL_INPUT and sometimes a subsequent download attempt is successful, so we should treat it as transient. Ideally we would behave as wget does in this case and retry the download picking up where we left off in the file rather than starting over, but that would require changes to libsoup I think. Sadly this patch does not fix the flatpak installation of spotify in the face of such errors, because flatpak doesn't use libostree to download extra data, but presumably it's possible we could encounter such an error pulling from an ostree repo, so the patch is still correct. --- src/libostree/ostree-fetcher-util.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index a9bd60a5..1c3f8104 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -202,6 +202,7 @@ _ostree_fetcher_should_retry_request (const GError *error, if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_TIMED_OUT) || g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) || g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) || + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_PARTIAL_INPUT) || #if !GLIB_CHECK_VERSION(2, 44, 0) g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) || #else From 635fd0aa9c418d2b7cd90f779db2208b0c70959d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 21 Mar 2020 20:39:18 +0000 Subject: [PATCH 007/177] tests/kola: Two test fixes Now that we're actually running this in CI. --- tests/kola/destructive/itest-label-selinux.sh | 1 + tests/kola/destructive/staged-deploy.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/kola/destructive/itest-label-selinux.sh b/tests/kola/destructive/itest-label-selinux.sh index 87fb26f0..7bfd2351 100755 --- a/tests/kola/destructive/itest-label-selinux.sh +++ b/tests/kola/destructive/itest-label-selinux.sh @@ -5,6 +5,7 @@ set -xeuo pipefail . ${KOLA_EXT_DATA}/libinsttest.sh +require_writable_sysroot date cd /ostree/repo/tmp diff --git a/tests/kola/destructive/staged-deploy.sh b/tests/kola/destructive/staged-deploy.sh index 47d7c9af..e0c17a27 100755 --- a/tests/kola/destructive/staged-deploy.sh +++ b/tests/kola/destructive/staged-deploy.sh @@ -71,6 +71,8 @@ case "${n}" in firstdeploycommit=$(rpm-ostree status |grep 'Commit:' |head -1|sed -e 's,^ *Commit: *,,') assert_streq "${firstdeploycommit}" "${newcommit}" # Cleanup + ## TODO remove workaround for https://github.com/coreos/rpm-ostree/pull/2021 + mkdir -p /var/lib/rpm-ostree/history rpm-ostree cleanup -rp echo "ok upgrade with staging" From a717535490001242ffcac55e745d702bb942c343 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 21 Mar 2020 17:54:13 +0000 Subject: [PATCH 008/177] main/commit: Rework control flow to use --tree=X path Rework the simple cases of "commit ." and "commit argv[1]" to generate the more general "--tree=X --tree=Y" path, so that we only have one primary control flow here. Prep for a future patch around loading SELinux policy from the first argument. --- src/ostree/ot-builtin-commit.c | 200 ++++++++++++++++----------------- 1 file changed, 99 insertions(+), 101 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 0eea09d7..16b07cb9 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -601,127 +601,125 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio mtree = ostree_mutable_tree_new (); - if (argc <= 1 && (opt_trees == NULL || opt_trees[0] == NULL)) + /* Convert implicit . or explicit path via argv into + * --tree=dir= so that we only have one primary code path below. + */ + if (opt_trees == NULL || opt_trees[0] == NULL) { - if (!ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, ".", mtree, modifier, - cancellable, error)) - goto out; + char *path; + if (argc <= 1) + path = "."; + else + path = argv[1]; + opt_trees = g_new0 (char *, 2); + opt_trees[0] = g_strconcat ("dir=", path, NULL); } - else if (opt_trees != NULL) + + const char *const*tree_iter; + const char *tree; + const char *eq; + g_assert (opt_trees && *opt_trees); + for (tree_iter = (const char *const*)opt_trees; *tree_iter; tree_iter++) { - const char *const*tree_iter; - const char *tree; - const char *eq; + tree = *tree_iter; - for (tree_iter = (const char *const*)opt_trees; *tree_iter; tree_iter++) + eq = strchr (tree, '='); + if (!eq) { - tree = *tree_iter; + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Missing type in tree specification '%s'", tree); + goto out; + } + g_free (tree_type); + tree_type = g_strndup (tree, eq - tree); + tree = eq + 1; - eq = strchr (tree, '='); - if (!eq) + g_clear_object (&object_to_commit); + if (strcmp (tree_type, "dir") == 0) + { + if (!ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, tree, mtree, modifier, + cancellable, error)) + goto out; + } + else if (strcmp (tree_type, "tar") == 0) + { + if (!opt_tar_pathname_filter) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Missing type in tree specification '%s'", tree); - goto out; - } - g_free (tree_type); - tree_type = g_strndup (tree, eq - tree); - tree = eq + 1; - - g_clear_object (&object_to_commit); - if (strcmp (tree_type, "dir") == 0) - { - if (!ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, tree, mtree, modifier, - cancellable, error)) - goto out; - } - else if (strcmp (tree_type, "tar") == 0) - { - if (!opt_tar_pathname_filter) + if (strcmp (tree, "-") == 0) { - if (strcmp (tree, "-") == 0) - { - if (!ostree_repo_write_archive_to_mtree_from_fd (repo, STDIN_FILENO, mtree, modifier, - opt_tar_autocreate_parents, - cancellable, error)) - goto out; - } - else - { - object_to_commit = g_file_new_for_path (tree); - - if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier, - opt_tar_autocreate_parents, - cancellable, error)) - goto out; - } + if (!ostree_repo_write_archive_to_mtree_from_fd (repo, STDIN_FILENO, mtree, modifier, + opt_tar_autocreate_parents, + cancellable, error)) + goto out; } else { -#ifdef HAVE_LIBARCHIVE - const char *comma = strchr (opt_tar_pathname_filter, ','); - if (!comma) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Missing ',' in --tar-pathname-filter"); - goto out; - } - const char *replacement = comma + 1; - g_autofree char *regexp_text = g_strndup (opt_tar_pathname_filter, comma - opt_tar_pathname_filter); - /* Use new API if we have a pathname filter */ - OstreeRepoImportArchiveOptions opts = { 0, }; - opts.autocreate_parents = opt_tar_autocreate_parents; - opts.translate_pathname = handle_translate_pathname; - g_autoptr(GRegex) regexp = g_regex_new (regexp_text, 0, 0, error); - TranslatePathnameData tpdata = { regexp, replacement }; - if (!regexp) - { - g_prefix_error (error, "--tar-pathname-filter: "); - goto out; - } - opts.translate_pathname_user_data = &tpdata; + object_to_commit = g_file_new_for_path (tree); - g_autoptr(OtAutoArchiveRead) archive; - if (strcmp (tree, "-") == 0) - archive = ot_open_archive_read_fd (STDIN_FILENO, error); - else - archive = ot_open_archive_read (tree, error); - - if (!archive) + if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier, + opt_tar_autocreate_parents, + cancellable, error)) goto out; - if (!ostree_repo_import_archive_to_mtree (repo, &opts, archive, mtree, - modifier, cancellable, error)) - goto out; -#else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "This version of ostree is not compiled with libarchive support"); - goto out; -#endif } } - else if (strcmp (tree_type, "ref") == 0) - { - if (!ostree_repo_read_commit (repo, tree, &object_to_commit, NULL, cancellable, error)) - goto out; - - if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, - cancellable, error)) - goto out; - } else { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid tree type specification '%s'", tree_type); +#ifdef HAVE_LIBARCHIVE + const char *comma = strchr (opt_tar_pathname_filter, ','); + if (!comma) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Missing ',' in --tar-pathname-filter"); + goto out; + } + const char *replacement = comma + 1; + g_autofree char *regexp_text = g_strndup (opt_tar_pathname_filter, comma - opt_tar_pathname_filter); + /* Use new API if we have a pathname filter */ + OstreeRepoImportArchiveOptions opts = { 0, }; + opts.autocreate_parents = opt_tar_autocreate_parents; + opts.translate_pathname = handle_translate_pathname; + g_autoptr(GRegex) regexp = g_regex_new (regexp_text, 0, 0, error); + TranslatePathnameData tpdata = { regexp, replacement }; + if (!regexp) + { + g_prefix_error (error, "--tar-pathname-filter: "); + goto out; + } + opts.translate_pathname_user_data = &tpdata; + + g_autoptr(OtAutoArchiveRead) archive; + if (strcmp (tree, "-") == 0) + archive = ot_open_archive_read_fd (STDIN_FILENO, error); + else + archive = ot_open_archive_read (tree, error); + + if (!archive) + goto out; + if (!ostree_repo_import_archive_to_mtree (repo, &opts, archive, mtree, + modifier, cancellable, error)) + goto out; +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); goto out; +#endif } } - } - else - { - g_assert (argc > 1); - if (!ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, argv[1], mtree, modifier, - cancellable, error)) - goto out; + else if (strcmp (tree_type, "ref") == 0) + { + if (!ostree_repo_read_commit (repo, tree, &object_to_commit, NULL, cancellable, error)) + goto out; + + if (!ostree_repo_write_directory_to_mtree (repo, object_to_commit, mtree, modifier, + cancellable, error)) + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid tree type specification '%s'", tree_type); + goto out; + } } if (mode_adds && g_hash_table_size (mode_adds) > 0) From ea16f7603d4eb2794d21bdd4ca8ccac4098682c5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 24 Mar 2020 12:24:46 +0000 Subject: [PATCH 009/177] tests/pull-repeated: Bump up retries to match max fails This test keeps occasionally failing in CI - as expected, because we retry 8 times for an object but it's completely possible for us to hit the <0.5% chance of 50% failure 8 times in a row. Since the max errors from the server is 100, set retries to the same thing. --- tests/test-pull-repeated.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-pull-repeated.sh b/tests/test-pull-repeated.sh index 237a863c..f0ea157e 100755 --- a/tests/test-pull-repeated.sh +++ b/tests/test-pull-repeated.sh @@ -92,8 +92,8 @@ pushd ${test_tmpdir} ostree_repo_init repo --mode=archive ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo -# Using 8 network retries gives error rate of <0.5%, when --random-408s=50 -${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=8 main +# We limit 408s above to 100, so 100 retries should be enough always. +${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=100 main echo "Success with big number of network retries" ${CMD_PREFIX} ostree --repo=repo fsck From b3bbbd154225e81980546b2c0b5ed98714830696 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 21 Mar 2020 14:48:24 +0000 Subject: [PATCH 010/177] repo/commit: Add support for --selinux-policy-from-base The [dev-overlay](https://github.com/coreos/coreos-assembler/blob/332c6ab3b91778d904224c3c960d9cc4739d60bd/src/cmd-dev-overlay) script shipped in coreos-assembler mostly exists to deal with the nontrivial logic around SELinux policy. Let's make the use case of "commit some binaries overlaying a base tree, using the base's selinux policy" just require a magical `--selinux-policy-from-base` argument to `ostree commit`. A new C API was added to implement this in the case of `--tree=ref`; when the base directory is already checked out, we can just reuse the existing logic that `--selinux-policy` was using. Requires: https://github.com/ostreedev/ostree/pull/2039 --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 2 +- src/libostree/ostree-repo-commit.c | 58 ++++++++++++++++++- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.h | 7 +++ src/ostree/ot-builtin-commit.c | 51 +++++++++++----- tests/kola/destructive/itest-label-selinux.sh | 17 ++++++ 7 files changed, 121 insertions(+), 16 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 32cf5228..3525d9f2 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -379,6 +379,7 @@ ostree_repo_commit_modifier_new OstreeRepoCommitModifierXattrCallback ostree_repo_commit_modifier_set_xattr_callback ostree_repo_commit_modifier_set_sepolicy +ostree_repo_commit_modifier_set_sepolicy_from_commit ostree_repo_commit_modifier_set_devino_cache ostree_repo_commit_modifier_ref ostree_repo_commit_modifier_unref diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index aa3392cc..3d5fd3bc 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -20,7 +20,7 @@ /* Add new symbols here. Release commits should copy this section into -released.sym. */ LIBOSTREE_2020.2 { global: - someostree_symbol_deleteme; + ostree_repo_commit_modifier_set_sepolicy_from_commit; } LIBOSTREE_2020.1; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index dac4573c..3f2f2beb 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -4227,9 +4227,11 @@ ostree_repo_commit_modifier_unref (OstreeRepoCommitModifier *modifier) if (modifier->xattr_destroy) modifier->xattr_destroy (modifier->xattr_user_data); - g_clear_object (&modifier->sepolicy); g_clear_pointer (&modifier->devino_cache, (GDestroyNotify)g_hash_table_unref); + g_clear_object (&modifier->sepolicy); + (void) glnx_tmpdir_delete (&modifier->sepolicy_tmpdir, NULL, NULL); + g_free (modifier); return; } @@ -4279,6 +4281,60 @@ ostree_repo_commit_modifier_set_sepolicy (OstreeRepoCommitModifier modifier->sepolicy = sepolicy ? g_object_ref (sepolicy) : NULL; } +/** + * ostree_repo_commit_modifier_set_sepolicy_from_commit: + * @modifier: Commit modifier + * @repo: OSTree repo containing @rev + * @rev: Find SELinux policy from this base commit + * @cancellable: + * @error: + * + * In many cases, one wants to create a "derived" commit from base commit. + * SELinux policy labels are part of that base commit. This API allows + * one to easily set up SELinux labeling from a base commit. + */ +gboolean +ostree_repo_commit_modifier_set_sepolicy_from_commit (OstreeRepoCommitModifier *modifier, + OstreeRepo *repo, + const char *rev, + GCancellable *cancellable, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR ("setting sepolicy from commit", error); + g_autofree char *commit = NULL; + g_autoptr(GFile) root = NULL; + if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error)) + return FALSE; + const char policypath[] = "usr/etc/selinux"; + g_autoptr(GFile) policyroot = g_file_get_child (root, policypath); + if (!g_file_query_exists (policyroot, NULL)) + return TRUE; /* No policy, nothing to do */ + + GLnxTmpDir tmpdir = {0,}; + if (!glnx_mkdtemp ("ostree-commit-sepolicy-XXXXXX", 0700, &tmpdir, error)) + return FALSE; + if (!glnx_shutil_mkdir_p_at (tmpdir.fd, "usr/etc", 0755, cancellable, error)) + return FALSE; + + OstreeRepoCheckoutAtOptions coopts = {0,}; + coopts.mode = OSTREE_REPO_CHECKOUT_MODE_USER; + coopts.subpath = glnx_strjoina ("/", policypath); + + if (!ostree_repo_checkout_at (repo, &coopts, tmpdir.fd, policypath, commit, cancellable, error)) + return glnx_prefix_error (error, "policy checkout"); + + g_autoptr(OstreeSePolicy) policy = ostree_sepolicy_new_at (tmpdir.fd, cancellable, error); + if (!policy) + return glnx_prefix_error (error, "reading policy"); + + ostree_repo_commit_modifier_set_sepolicy (modifier, policy); + /* Transfer ownership */ + modifier->sepolicy_tmpdir = tmpdir; + tmpdir.initialized = FALSE; + + return TRUE; +} + /** * ostree_repo_commit_modifier_set_devino_cache: * @modifier: Modifier diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 571cab6b..9f722a39 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -76,6 +76,7 @@ struct OstreeRepoCommitModifier { GDestroyNotify xattr_destroy; gpointer xattr_user_data; + GLnxTmpDir sepolicy_tmpdir; OstreeSePolicy *sepolicy; GHashTable *devino_cache; }; diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 40d3f773..014afff9 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -691,6 +691,13 @@ _OSTREE_PUBLIC void ostree_repo_commit_modifier_set_sepolicy (OstreeRepoCommitModifier *modifier, OstreeSePolicy *sepolicy); +_OSTREE_PUBLIC +gboolean ostree_repo_commit_modifier_set_sepolicy_from_commit (OstreeRepoCommitModifier *modifier, + OstreeRepo *repo, + const char *commit, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC void ostree_repo_commit_modifier_set_devino_cache (OstreeRepoCommitModifier *modifier, OstreeRepoDevInoCache *cache); diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 16b07cb9..4cca56d0 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -53,6 +53,7 @@ static gboolean opt_tar_autocreate_parents; static char *opt_tar_pathname_filter; static gboolean opt_no_xattrs; static char *opt_selinux_policy; +static gboolean opt_selinux_policy_from_base; static gboolean opt_canonical_permissions; static gboolean opt_consume; static gboolean opt_devino_canonical; @@ -107,6 +108,7 @@ static GOptionEntry options[] = { { "canonical-permissions", 0, 0, G_OPTION_ARG_NONE, &opt_canonical_permissions, "Canonicalize permissions in the same way bare-user does for hardlinked files", NULL }, { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Do not import extended attributes", NULL }, { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /)", "PATH" }, + { "selinux-policy-from-base", 'P', 0, G_OPTION_ARG_NONE, &opt_selinux_policy_from_base, "Set SELinux labels based on first --tree argument", NULL }, { "link-checkout-speedup", 0, 0, G_OPTION_ARG_NONE, &opt_link_checkout_speedup, "Optimize for commits of trees composed of hardlinks into the repository", NULL }, { "devino-canonical", 'I', 0, G_OPTION_ARG_NONE, &opt_devino_canonical, "Assume hardlinked objects are unmodified. Implies --link-checkout-speedup", NULL }, { "tar-autocreate-parents", 0, 0, G_OPTION_ARG_NONE, &opt_tar_autocreate_parents, "When loading tar archives, automatically create parent directories as needed", NULL }, @@ -550,6 +552,11 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES; if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); + if (opt_selinux_policy && opt_selinux_policy_from_base) + { + glnx_throw (error, "Cannot specify both --selinux-policy and --selinux-policy-from-base"); + goto out; + } if (flags != 0 || opt_owner_uid >= 0 @@ -557,25 +564,13 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio || opt_statoverride_file != NULL || opt_skiplist_file != NULL || opt_no_xattrs - || opt_selinux_policy) + || opt_selinux_policy + || opt_selinux_policy_from_base) { filter_data.mode_adds = mode_adds; filter_data.skip_list = skip_list; modifier = ostree_repo_commit_modifier_new (flags, commit_filter, &filter_data, NULL); - if (opt_selinux_policy) - { - glnx_autofd int rootfs_dfd = -1; - if (!glnx_opendirat (AT_FDCWD, opt_selinux_policy, TRUE, &rootfs_dfd, error)) - { - g_prefix_error (error, "selinux-policy: "); - goto out; - } - policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error); - if (!policy) - goto out; - ostree_repo_commit_modifier_set_sepolicy (modifier, policy); - } } if (opt_editor) @@ -621,6 +616,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio g_assert (opt_trees && *opt_trees); for (tree_iter = (const char *const*)opt_trees; *tree_iter; tree_iter++) { + const gboolean first = (tree_iter == (const char *const*)opt_trees); tree = *tree_iter; eq = strchr (tree, '='); @@ -637,12 +633,33 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio g_clear_object (&object_to_commit); if (strcmp (tree_type, "dir") == 0) { + if (first && opt_selinux_policy_from_base) + { + opt_selinux_policy = g_strdup (tree); + opt_selinux_policy_from_base = FALSE; + } + if (first && opt_selinux_policy) + { + g_assert (modifier); + glnx_autofd int rootfs_dfd = -1; + if (!glnx_opendirat (AT_FDCWD, opt_selinux_policy, TRUE, &rootfs_dfd, error)) + goto out; + policy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error); + if (!policy) + goto out; + ostree_repo_commit_modifier_set_sepolicy (modifier, policy); + } if (!ostree_repo_write_dfd_to_mtree (repo, AT_FDCWD, tree, mtree, modifier, cancellable, error)) goto out; } else if (strcmp (tree_type, "tar") == 0) { + if (first && opt_selinux_policy_from_base) + { + glnx_throw (error, "Cannot use --selinux-policy-from-base with tar"); + goto out; + } if (!opt_tar_pathname_filter) { if (strcmp (tree, "-") == 0) @@ -707,6 +724,12 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio } else if (strcmp (tree_type, "ref") == 0) { + if (first && opt_selinux_policy_from_base) + { + g_assert (modifier); + if (!ostree_repo_commit_modifier_set_sepolicy_from_commit (modifier, repo, tree, cancellable, error)) + goto out; + } if (!ostree_repo_read_commit (repo, tree, &object_to_commit, NULL, cancellable, error)) goto out; diff --git a/tests/kola/destructive/itest-label-selinux.sh b/tests/kola/destructive/itest-label-selinux.sh index 7bfd2351..d7337124 100755 --- a/tests/kola/destructive/itest-label-selinux.sh +++ b/tests/kola/destructive/itest-label-selinux.sh @@ -6,6 +6,7 @@ set -xeuo pipefail . ${KOLA_EXT_DATA}/libinsttest.sh require_writable_sysroot +prepare_tmpdir /var/tmp date cd /ostree/repo/tmp @@ -87,3 +88,19 @@ rm co -rf ostree refs --delete testbranch echo "ok checkout selinux and skip-list" date + +mkdir -p usr/{bin,lib,etc} +echo 'somebinary' > usr/bin/somebinary +ls -Z usr/bin/somebinary > lsz.txt +assert_not_file_has_content lsz.txt ':bin_t:' +rm -f lsz.txt +echo 'somelib' > usr/lib/somelib.so +echo 'someconf' > usr/etc/some.conf +ostree commit -b newbase --selinux-policy-from-base --tree=ref=${host_refspec} --tree=dir=$(pwd) +ostree ls -X newbase /usr/bin/somebinary > newls.txt +assert_file_has_content newls.txt ':bin_t:' +ostree ls -X newbase /usr/lib/somelib.so > newls.txt +assert_file_has_content newls.txt ':lib_t:' +ostree ls -X newbase /usr/etc/some.conf > newls.txt +assert_file_has_content newls.txt ':etc_t:' +echo "ok commit --selinux-policy-from-base" From 84c8164610ee3df9bbd06f0be9e37a873708ec2d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sat, 15 Jun 2019 20:10:47 +0300 Subject: [PATCH 011/177] Add libsodium dependency Allow to configure with libsodium flag. Signed-off-by: Denis Pynkin --- Makefile-ostree.am | 4 ++++ configure.ac | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index f861afe4..470d23d3 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -112,6 +112,10 @@ ostree_SOURCES += \ $(NULL) endif +if USE_LIBSODIUM +ostree_CFLAGS += $(OT_DEP_LIBSODIUM_CFLAGS) +ostree_LDADD += $(OT_DEP_LIBSODIUM_LIBS) +endif # USE_LIBSODIUM if USE_CURL_OR_SOUP ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ diff --git a/configure.ac b/configure.ac index e2d867e7..e41ccb70 100644 --- a/configure.ac +++ b/configure.ac @@ -242,6 +242,21 @@ dnl to link to it directly. ) AM_CONDITIONAL(USE_GPGME, test "x$have_gpgme" = xyes) + +LIBSODIUM_DEPENDENCY="1.0.14" +AC_ARG_WITH(libsodium, + AS_HELP_STRING([--with-libsodium], [Use libsodium @<:@default=no@:>@]), + [], [with_libsodium=no]) +AS_IF([test x$with_libsodium != xno], [ + AC_DEFINE([HAVE_LIBSODIUM], 1, [Define if using libsodium]) + PKG_CHECK_MODULES(OT_DEP_LIBSODIUM, libsodium >= $LIBSODIUM_DEPENDENCY, have_libsodium=yes, have_libsodium=no) + AS_IF([ test x$have_libsodium = xno ], [ + AC_MSG_ERROR([Need LIBSODIUM version $LIBSODIUM_DEPENDENCY or later]) + ]) + OSTREE_FEATURES="$OSTREE_FEATURES libsodium" +], with_libsodium=no ) +AM_CONDITIONAL(USE_LIBSODIUM, test "x$have_libsodium" = xyes) + LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" # What's in RHEL7.2. FUSE_DEPENDENCY="fuse >= 2.9.2" From edbbe1c4f2267c0d95d4ee14f6cbc516a1d31dca Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 29 Jul 2019 02:32:28 +0300 Subject: [PATCH 012/177] lib/sign: initial implementation Added the initial version of signing interface allowing to allowing to sign and verify commits. Implemented initial signing modules: - dummy -- simple module allowing to sign/verify with ASCII string - ed25519 -- module allowing to sign/verify commit with ed25519 (EdDSA) signature scheme provided by libsodium library. Signed-off-by: Denis Pynkin --- Makefile-libostree-defines.am | 2 + Makefile-libostree.am | 14 ++ Makefile-ostree.am | 10 +- apidoc/ostree-sections.txt | 23 ++ src/libostree/libostree-devel.sym | 16 ++ src/libostree/ostree-sign-dummy.c | 181 +++++++++++++++ src/libostree/ostree-sign-dummy.h | 63 +++++ src/libostree/ostree-sign-ed25519.c | 342 +++++++++++++++++++++++++++ src/libostree/ostree-sign-ed25519.h | 75 ++++++ src/libostree/ostree-sign.c | 347 ++++++++++++++++++++++++++++ src/libostree/ostree-sign.h | 156 +++++++++++++ 11 files changed, 1224 insertions(+), 5 deletions(-) create mode 100644 src/libostree/ostree-sign-dummy.c create mode 100644 src/libostree/ostree-sign-dummy.h create mode 100644 src/libostree/ostree-sign-ed25519.c create mode 100644 src/libostree/ostree-sign-ed25519.h create mode 100644 src/libostree/ostree-sign.c create mode 100644 src/libostree/ostree-sign.h diff --git a/Makefile-libostree-defines.am b/Makefile-libostree-defines.am index 06035157..43e09281 100644 --- a/Makefile-libostree-defines.am +++ b/Makefile-libostree-defines.am @@ -46,6 +46,8 @@ libostree_public_headers = \ src/libostree/ostree-repo-finder-mount.h \ src/libostree/ostree-repo-finder-override.h \ src/libostree/ostree-kernel-args.h \ + src/libostree/ostree-sign.h \ + src/libostree/ostree-sign-ed25519.h \ $(NULL) # This one is generated via configure.ac, and the gtk-doc diff --git a/Makefile-libostree.am b/Makefile-libostree.am index a7e7e123..c0a7ac9f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -262,6 +262,20 @@ libostree_1_la_CFLAGS += $(OT_DEP_SELINUX_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_SELINUX_LIBS) endif +libostree_1_la_SOURCES += \ + src/libostree/ostree-sign.c \ + src/libostree/ostree-sign.h \ + src/libostree/ostree-sign-dummy.c \ + src/libostree/ostree-sign-dummy.h \ + src/libostree/ostree-sign-ed25519.c \ + src/libostree/ostree-sign-ed25519.h \ + $(NULL) + +if USE_LIBSODIUM +libostree_1_la_CFLAGS += $(OT_DEP_LIBSODIUM_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_LIBSODIUM_LIBS) +endif # USE_LIBSODIUM + # XXX: work around clang being passed -fstack-clash-protection which it doesn't understand # See: https://bugzilla.redhat.com/show_bug.cgi?id=1672012 INTROSPECTION_SCANNER_ENV = CC=gcc diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 470d23d3..f37d974a 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -112,11 +112,6 @@ ostree_SOURCES += \ $(NULL) endif -if USE_LIBSODIUM -ostree_CFLAGS += $(OT_DEP_LIBSODIUM_CFLAGS) -ostree_LDADD += $(OT_DEP_LIBSODIUM_LIBS) -endif # USE_LIBSODIUM - if USE_CURL_OR_SOUP ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ src/ostree/ot-remote-builtin-delete-cookie.c \ @@ -166,3 +161,8 @@ if USE_LIBARCHIVE ostree_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) ostree_LDADD += $(OT_DEP_LIBARCHIVE_LIBS) endif + +if USE_LIBSODIUM +ostree_CFLAGS += $(OT_DEP_LIBSODIUM_CFLAGS) +ostree_LDADD += $(OT_DEP_LIBSODIUM_LIBS) +endif # USE_LIBSODIUM diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 3525d9f2..cfc4a340 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -705,3 +705,26 @@ ostree_kernel_args_from_string ostree_kernel_args_to_strv ostree_kernel_args_to_string + +
+ostree-sign +OstreeSign +OstreeSignDummy +OstreeSignEd25519 +ostree_sign_list_names +ostree_sign_commit +ostree_sign_commit_verify +ostree_sign_data +ostree_sign_get_by_name +ostree_sign_get_name +ostree_sign_detached_metadata_append +ostree_sign_metadata_verify +ostree_sign_load_pk +ostree_sign_set_pk +ostree_sign_set_sk +ostree_sign_ed25519_keypair_generate + +ostree_sign_get_type +ostree_sign_dummy_get_type +ostree_sign_ed25519_get_type +
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 3d5fd3bc..4066f383 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -21,6 +21,22 @@ LIBOSTREE_2020.2 { global: ostree_repo_commit_modifier_set_sepolicy_from_commit; + someostree_symbol_deleteme; + ostree_sign_get_type; + ostree_sign_list_names; + ostree_sign_commit; + ostree_sign_commit_verify; + ostree_sign_data; + ostree_sign_get_by_name; + ostree_sign_get_name; + ostree_sign_detached_metadata_append; + ostree_sign_metadata_verify; + ostree_sign_load_pk; + ostree_sign_set_pk; + ostree_sign_set_sk; + ostree_sign_dummy_get_type; + ostree_sign_ed25519_get_type; + ostree_sign_ed25519_keypair_generate; } LIBOSTREE_2020.1; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c new file mode 100644 index 00000000..e489a988 --- /dev/null +++ b/src/libostree/ostree-sign-dummy.c @@ -0,0 +1,181 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "config.h" + +#include "ostree-sign-dummy.h" +#include + +#define OSTREE_SIGN_DUMMY_NAME "dummy" + +#define OSTREE_SIGN_METADATA_DUMMY_KEY "ostree.sign.dummy" +#define OSTREE_SIGN_METADATA_DUMMY_TYPE "aay" + +#define OSTREE_SIGN_DUMMY_SIGNATURE "dummysign" + +struct _OstreeSignDummy +{ + GObject parent; + gchar *signature_ascii; +}; + +static void +ostree_sign_dummy_iface_init (OstreeSignInterface *self); + +G_DEFINE_TYPE_WITH_CODE (OstreeSignDummy, ostree_sign_dummy, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_SIGN, ostree_sign_dummy_iface_init)); + +static void +ostree_sign_dummy_iface_init (OstreeSignInterface *self) +{ + g_debug ("%s enter", __FUNCTION__); + + self->data = ostree_sign_dummy_data; + self->get_name = ostree_sign_dummy_get_name; + self->metadata_key = ostree_sign_dummy_metadata_key; + self->metadata_format = ostree_sign_dummy_metadata_format; + self->metadata_verify = ostree_sign_dummy_metadata_verify; + self->set_sk = ostree_sign_dummy_set_signature; + self->set_pk = ostree_sign_dummy_set_signature; +} + +static void +ostree_sign_dummy_class_init (OstreeSignDummyClass *self) +{ + g_debug ("%s enter", __FUNCTION__); + GObjectClass *object_class = G_OBJECT_CLASS(self); +} + +static void +ostree_sign_dummy_init (OstreeSignDummy *self) +{ + g_debug ("%s enter", __FUNCTION__); + + self->signature_ascii = g_strdup(OSTREE_SIGN_DUMMY_SIGNATURE); +} + +gboolean ostree_sign_dummy_set_signature (OstreeSign *self, GVariant *key, GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + + if (sign->signature_ascii != NULL) + g_free(sign->signature_ascii); + + sign->signature_ascii = g_variant_dup_string (key, 0); + + return TRUE; +} + +gboolean ostree_sign_dummy_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error) +{ + + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + + *signature = g_bytes_new (sign->signature_ascii, strlen(sign->signature_ascii)); + + return TRUE; +} + +gchar * ostree_sign_dummy_get_name (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + g_autofree gchar *name = g_strdup(OSTREE_SIGN_DUMMY_NAME); + + return g_steal_pointer (&name); +} + +gchar * ostree_sign_dummy_metadata_key (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_autofree gchar *key = g_strdup(OSTREE_SIGN_METADATA_DUMMY_KEY); + return g_steal_pointer (&key); +} + +gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_autofree gchar *type = g_strdup(OSTREE_SIGN_METADATA_DUMMY_TYPE); + return g_steal_pointer (&type); +} + +gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (data != NULL, FALSE); + + OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + + gboolean ret = FALSE; + + if (signatures == NULL) + { + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: dummy: commit have no signatures of my type"); + goto err; + } + + + if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_DUMMY_TYPE)) + { + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: dummy: wrong type passed for verification"); + goto err; + } + + for (gsize i = 0; i < g_variant_n_children(signatures); i++) + { + g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); + g_autoptr (GBytes) signature = g_variant_get_data_as_bytes(child); + + gsize sign_size = 0; + g_bytes_get_data (signature, &sign_size); + g_autofree gchar *sign_ascii = g_strndup(g_bytes_get_data (signature, NULL), sign_size); + g_debug("Read signature %d: %s", (gint)i, sign_ascii); + + if (!g_strcmp0(sign_ascii, sign->signature_ascii)) + ret = TRUE; + } + +err: + return ret; +} diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h new file mode 100644 index 00000000..8bbd407d --- /dev/null +++ b/src/libostree/ostree-sign-dummy.h @@ -0,0 +1,63 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Denis Pynkin (d4s) + */ + +#pragma once + +#include "ostree-sign.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_SIGN_DUMMY (ostree_sign_dummy_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeSignDummy, + ostree_sign_dummy, + OSTREE, + SIGN_DUMMY, + GObject) + +gchar * ostree_sign_dummy_get_name (OstreeSign *self); + +gboolean ostree_sign_dummy_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); + +gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); +gchar * ostree_sign_dummy_metadata_format (OstreeSign *self); + +gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); + +gboolean ostree_sign_dummy_set_signature (OstreeSign *self, GVariant *key, GError **error); + +void ostree_sign_dummy_finalize (GObject *gobject); + +G_END_DECLS + diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c new file mode 100644 index 00000000..e3ab6b57 --- /dev/null +++ b/src/libostree/ostree-sign-ed25519.c @@ -0,0 +1,342 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Denis Pynkin (d4s) + */ + +#include "config.h" + +#include "ostree-sign-ed25519.h" +#ifdef HAVE_LIBSODIUM +#include +#endif + +#define OSTREE_SIGN_ED25519_NAME "ed25519" + +#define OSTREE_SIGN_METADATA_ED25519_KEY "ostree.sign.ed25519" +#define OSTREE_SIGN_METADATA_ED25519_TYPE "aay" + +struct _OstreeSignEd25519 +{ + GObject parent; + gboolean initialized; + guchar *secret_key; + guchar *public_key; +}; + +static void +ostree_sign_ed25519_iface_init (OstreeSignInterface *self); + +G_DEFINE_TYPE_WITH_CODE (OstreeSignEd25519, ostree_sign_ed25519, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_SIGN, ostree_sign_ed25519_iface_init)); + +static void +ostree_sign_ed25519_iface_init (OstreeSignInterface *self) +{ + g_debug ("%s enter", __FUNCTION__); + + self->data = ostree_sign_ed25519_data; + self->get_name = ostree_sign_ed25519_get_name; + self->metadata_key = ostree_sign_ed25519_metadata_key; + self->metadata_format = ostree_sign_ed25519_metadata_format; + self->metadata_verify = ostree_sign_ed25519_metadata_verify; + self->set_sk = ostree_sign_ed25519_set_sk; + self->set_pk = ostree_sign_ed25519_set_pk; +} + +static void +ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) +{ + g_debug ("%s enter", __FUNCTION__); + GObjectClass *object_class = G_OBJECT_CLASS(self); +} + +static void +ostree_sign_ed25519_init (OstreeSignEd25519 *self) +{ + g_debug ("%s enter", __FUNCTION__); + + self->initialized = TRUE; + self->secret_key = NULL; + self->public_key = NULL; + +#ifdef HAVE_LIBSODIUM + if (sodium_init() < 0) + { + self->initialized = FALSE; + g_warning ("libsodium library couldn't be initialized"); + } +#else + g_error ("ed25519 signature isn't supported"); +#endif /* HAVE_LIBSODIUM */ +} + +gboolean ostree_sign_ed25519_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error) +{ + + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + +#ifdef HAVE_LIBSODIUM + g_autofree guchar *sig = NULL; +#endif + + if ((sign->initialized != TRUE) || (sign->secret_key == NULL)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to sign: libsodium library isn't initialized properly"); + goto err; + } +#ifdef HAVE_LIBSODIUM + unsigned long long sig_size = 0; + + sig = g_malloc0(crypto_sign_BYTES); + + if (crypto_sign_detached (sig, + &sig_size, + g_bytes_get_data (data, NULL), + g_bytes_get_size (data), + sign->secret_key)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to sign the object"); + goto err; + } + + g_debug ("sign: data hash = 0x%x", g_bytes_hash(data)); + *signature = g_bytes_new (sig, sig_size); + return TRUE; +#endif /* HAVE_LIBSODIUM */ +err: + return FALSE; +} + +gchar * ostree_sign_ed25519_get_name (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + g_autofree gchar *name = g_strdup (OSTREE_SIGN_ED25519_NAME); + + return g_steal_pointer (&name); +} + +gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_autofree gchar *key = g_strdup(OSTREE_SIGN_METADATA_ED25519_KEY); + return g_steal_pointer (&key); +} + +gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_autofree gchar *type = g_strdup (OSTREE_SIGN_METADATA_ED25519_TYPE); + return g_steal_pointer (&type); +} + +gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (data != NULL, FALSE); + gboolean ret = FALSE; + + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + if (signatures == NULL) + { + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: ed25519: commit have no signatures of my type"); + goto err; + } + + if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_ED25519_TYPE)) + { + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: ed25519: wrong type passed for verification"); + goto err; + } + + if ((sign->initialized != TRUE) || (sign->public_key == NULL)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to verify: libsodium library isn't initialized properly"); + goto err; + } + +#ifdef HAVE_LIBSODIUM + g_debug ("verify: data hash = 0x%x", g_bytes_hash(data)); + + for (gsize i = 0; i < g_variant_n_children(signatures); i++) + { + g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); + g_autoptr (GBytes) signature = g_variant_get_data_as_bytes(child); + + g_autofree char * hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + + g_debug("Read signature %d: %s", (gint)i, g_variant_print(child, TRUE)); + + if (crypto_sign_verify_detached ((guchar *) g_variant_get_data (child), + g_bytes_get_data (data, NULL), + g_bytes_get_size (data), + sign->public_key) != 0) + { + /* Incorrect signature! */ + g_debug("Signature couldn't be verified with key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, crypto_sign_PUBLICKEYBYTES)); + } + else + { + ret = TRUE; + g_debug ("Signature verified successfully with key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, crypto_sign_PUBLICKEYBYTES)); + } + } + + if (ret != TRUE) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to verify: no valid signatures found"); +#endif /* HAVE_LIBSODIUM */ + + return ret; +err: + return FALSE; +} + +gboolean +ostree_sign_ed25519_keypair_generate (OstreeSign *self, + GVariant **out_secret_key, + GVariant **out_public_key, + GError **error) + { + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + if (sign->initialized != TRUE) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to sign -- libsodium library isn't initialized properly"); + goto err; + } + +#ifdef HAVE_LIBSODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + + if (crypto_sign_keypair(pk, sk)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not able to generate keypair"); + goto err; + } + + *out_secret_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, sk, crypto_sign_SECRETKEYBYTES, sizeof(guchar)); + *out_public_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, pk, crypto_sign_PUBLICKEYBYTES, sizeof(guchar)); + + return TRUE; +#endif /* HAVE_LIBSODIUM */ + +err: + return FALSE; +} + +gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, + GVariant *secret_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + +#ifdef HAVE_LIBSODIUM + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + g_autofree char * hex = NULL; + + g_free (sign->secret_key); + + gsize n_elements = 0; + sign->secret_key = (guchar *) g_variant_get_fixed_array (secret_key, &n_elements, sizeof(guchar)); + + if (n_elements != crypto_sign_SECRETKEYBYTES) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Incorrect ed25519 secret key"); + goto err; + } + + hex = g_malloc0 (crypto_sign_SECRETKEYBYTES*2 + 1); + g_debug ("Set ed25519 secret key = %s", sodium_bin2hex (hex, crypto_sign_SECRETKEYBYTES*2+1, sign->secret_key, n_elements)); + + return TRUE; + +err: +#endif /* HAVE_LIBSODIUM */ + return FALSE; +} + +gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, + GVariant *public_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + +#ifdef HAVE_LIBSODIUM + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + g_autofree char * hex = NULL; + + gsize n_elements = 0; + g_free (sign->public_key); + sign->public_key = (guchar *) g_variant_get_fixed_array (public_key, &n_elements, sizeof(guchar)); + + hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, n_elements)); + + if (n_elements != crypto_sign_PUBLICKEYBYTES) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Incorrect ed25519 public key"); + goto err; + } + + g_debug ("Set ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, n_elements)); + + return TRUE; + +err: +#endif /* HAVE_LIBSODIUM */ + return FALSE; +} diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h new file mode 100644 index 00000000..d4a6b56d --- /dev/null +++ b/src/libostree/ostree-sign-ed25519.h @@ -0,0 +1,75 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Denis Pynkin (d4s) + */ + +#pragma once + +#include "ostree-sign.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_SIGN_ED25519 (ostree_sign_ed25519_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeSignEd25519, + ostree_sign_ed25519, + OSTREE, + SIGN_ED25519, + GObject) + + +gboolean ostree_sign_ed25519_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); + +gchar * ostree_sign_ed25519_get_name (OstreeSign *self); +gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); +gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); + +gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); + +gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, + GVariant *secret_key, + GError **error); + +gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, + GVariant *public_key, + GError **error); + +void ostree_sign_ed25519_finalize (GObject *gobject); + +_OSTREE_PUBLIC +gboolean ostree_sign_ed25519_keypair_generate (OstreeSign *self, + GVariant **out_secret_key, + GVariant **out_public_key, + GError **error); + +G_END_DECLS + diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c new file mode 100644 index 00000000..96455f86 --- /dev/null +++ b/src/libostree/ostree-sign.c @@ -0,0 +1,347 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "config.h" + +#include +#include +#include +#include "libglnx.h" +#include "otutil.h" + +#include "ostree-autocleanups.h" +#include "ostree-core.h" +#include "ostree-sign.h" +#include "ostree-sign-dummy.h" +#ifdef HAVE_LIBSODIUM +#include "ostree-sign-ed25519.h" +#endif + +#define G_LOG_DOMAIN "OSTreeSign" + +G_DEFINE_INTERFACE (OstreeSign, ostree_sign, G_TYPE_OBJECT) + +static void +ostree_sign_default_init (OstreeSignInterface *iface) +{ + g_debug ("OstreeSign initialization"); +} + +gchar * ostree_sign_metadata_key (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_key != NULL, NULL); + return OSTREE_SIGN_GET_IFACE (self)->metadata_key (self); +} + +gchar * ostree_sign_metadata_format (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_format != NULL, NULL); + return OSTREE_SIGN_GET_IFACE (self)->metadata_format (self); +} + +gboolean ostree_sign_set_sk (OstreeSign *self, + GVariant *secret_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + if (OSTREE_SIGN_GET_IFACE (self)->set_sk == NULL) + return TRUE; + + return OSTREE_SIGN_GET_IFACE (self)->set_sk (self, secret_key, error); +} + +gboolean ostree_sign_set_pk (OstreeSign *self, + GVariant *public_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + if (OSTREE_SIGN_GET_IFACE (self)->set_pk == NULL) + return TRUE; + + return OSTREE_SIGN_GET_IFACE (self)->set_pk (self, public_key, error); +} + +/* Load private keys for verification from anywhere. + * No need to have the same function for secret keys -- the signing SW must do it in it's own way + * */ +gboolean +ostree_sign_load_pk (OstreeSign *self, + gchar *remote_name, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->load_pk != NULL, FALSE); + + if (remote_name == NULL) + remote_name = OSTREE_SIGN_ALL_REMOTES; + + return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, remote_name, error); +} + +gboolean ostree_sign_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error) +{ + + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data != NULL, FALSE); + + return OSTREE_SIGN_GET_IFACE (self)->data (self, data, signature, cancellable, error); +} + +/* + * Adopted version of _ostree_detached_metadata_append_gpg_sig () + */ +GVariant * +ostree_sign_detached_metadata_append (OstreeSign *self, + GVariant *existing_metadata, + GBytes *signature_bytes) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (signature_bytes != NULL, FALSE); + + GVariantDict metadata_dict; + g_autoptr(GVariant) signature_data = NULL; + g_autoptr(GVariantBuilder) signature_builder = NULL; + + g_variant_dict_init (&metadata_dict, existing_metadata); + + g_autofree gchar *signature_key = ostree_sign_metadata_key(self); + g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); + + signature_data = g_variant_dict_lookup_value (&metadata_dict, + signature_key, + (GVariantType*)signature_format); + + /* signature_data may be NULL */ + signature_builder = ot_util_variant_builder_from_variant (signature_data, signature_format); + + g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes)); + + g_variant_dict_insert_value (&metadata_dict, + signature_key, + g_variant_builder_end (signature_builder)); + + return g_variant_dict_end (&metadata_dict); +} + + +gboolean +ostree_sign_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_verify != NULL, FALSE); + + return OSTREE_SIGN_GET_IFACE (self)->metadata_verify(self, data, signatures, error); +} + +gboolean +ostree_sign_commit_verify (OstreeSign *self, + OstreeRepo *repo, + const gchar *commit_checksum, + GCancellable *cancellable, + GError **error) + +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + g_autoptr(GVariant) commit_variant = NULL; + /* Load the commit */ + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, + commit_checksum, &commit_variant, + error)) + return glnx_prefix_error (error, "Failed to read commit"); + + /* Load the metadata */ + g_autoptr(GVariant) metadata = NULL; + if (!ostree_repo_read_commit_detached_metadata (repo, + commit_checksum, + &metadata, + cancellable, + error)) + return glnx_prefix_error (error, "Failed to read detached metadata"); + + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit_variant); + + /* XXX This is a hackish way to indicate to use ALL remote-specific + * keyrings in the signature verification. We want this when + * verifying a signed commit that's already been pulled. */ +/* + if (remote_name == NULL) + remote_name = OSTREE_ALL_REMOTES; +*/ + + g_autoptr(GVariant) signatures = NULL; + + g_autofree gchar *signature_key = ostree_sign_metadata_key(self); + g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); + + if (metadata) + signatures = g_variant_lookup_value (metadata, + signature_key, + signature_format); + + + return ostree_sign_metadata_verify (self, + signed_data, + signatures, + error); +} + +const gchar * ostree_sign_get_name (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->get_name != NULL, FALSE); + + return OSTREE_SIGN_GET_IFACE (self)->get_name (self); +} + +OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + GType types [] = { +#if defined(HAVE_LIBSODIUM) + OSTREE_TYPE_SIGN_ED25519, +#endif + OSTREE_TYPE_SIGN_DUMMY + }; + OstreeSign *ret = NULL; + + for (gint i=0; i < G_N_ELEMENTS(types); i++) + { + g_autoptr (OstreeSign) sign = g_object_new (types[i], NULL); + g_autofree gchar *sign_name = OSTREE_SIGN_GET_IFACE (sign)->get_name(sign); + + g_debug ("Found '%s' signing module", sign_name); + + if (g_strcmp0 (name, sign_name) == 0) + { + ret = g_steal_pointer (&sign); + break; + } + } + + if (ret == NULL) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Requested signature type is not implemented"); + + return ret; +} + + +/** + * ostree_sign_commit: + * @self: Self + * @commit_checksum: SHA256 of given commit to sign + * @cancellable: A #GCancellable + * @error: a #GError + * + * Add a GPG signature to a commit. + */ +gboolean +ostree_sign_commit (OstreeSign *self, + OstreeRepo *repo, + const gchar *commit_checksum, + GCancellable *cancellable, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + g_autoptr(GBytes) commit_data = NULL; + g_autoptr(GBytes) signature = NULL; + g_autoptr(GVariant) commit_variant = NULL; + g_autoptr(GVariant) old_metadata = NULL; + g_autoptr(GVariant) new_metadata = NULL; + + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, + commit_checksum, &commit_variant, error)) + return glnx_prefix_error (error, "Failed to read commit"); + + if (!ostree_repo_read_commit_detached_metadata (repo, + commit_checksum, + &old_metadata, + cancellable, + error)) + return glnx_prefix_error (error, "Failed to read detached metadata"); + + // TODO: d4s: check if already signed? + + commit_data = g_variant_get_data_as_bytes (commit_variant); + + if (!ostree_sign_data (self, commit_data, &signature, + cancellable, error)) + return glnx_prefix_error (error, "Not able to sign the cobject"); + + new_metadata = + ostree_sign_detached_metadata_append (self, old_metadata, signature); + + if (!ostree_repo_write_commit_detached_metadata (repo, + commit_checksum, + new_metadata, + cancellable, + error)) + return FALSE; + + return TRUE; +} + +GStrv ostree_sign_list_names(void) +{ + g_debug ("%s enter", __FUNCTION__); + + GType types [] = { +#if defined(HAVE_LIBSODIUM) + OSTREE_TYPE_SIGN_ED25519, +#endif + OSTREE_TYPE_SIGN_DUMMY + }; + GStrv names = g_new0 (char *, G_N_ELEMENTS(types)+1); + gint i = 0; + + for (i=0; i < G_N_ELEMENTS(types); i++) + { + g_autoptr (OstreeSign) sign = g_object_new (types[i], NULL); + names[i] = OSTREE_SIGN_GET_IFACE (sign)->get_name(sign); + g_debug ("Found '%s' signing module", names[i]); + } + + return names; +} diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h new file mode 100644 index 00000000..f06206aa --- /dev/null +++ b/src/libostree/ostree-sign.h @@ -0,0 +1,156 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright © 2019 Collabora Ltd. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Denis Pynkin (d4s) + */ + +#pragma once + +#include +#include + +#include "ostree-ref.h" +#include "ostree-remote.h" +#include "ostree-types.h" + +/* Special remote */ +#define OSTREE_SIGN_ALL_REMOTES "__OSTREE_ALL_REMOTES__" + + +G_BEGIN_DECLS + +#define OSTREE_TYPE_SIGN (ostree_sign_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_INTERFACE (OstreeSign, ostree_sign, OSTREE, SIGN, GObject) + +struct _OstreeSignInterface +{ + GTypeInterface g_iface; + gchar *(* get_name) (OstreeSign *self); + gboolean (* data) (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); + gchar *(* metadata_key) (OstreeSign *self); + gchar *(* metadata_format) (OstreeSign *self); + gboolean (* metadata_verify) (OstreeSign *self, + GBytes *data, + GVariant *metadata, + GError **error); + + gboolean (* set_sk) (OstreeSign *self, + GVariant *secret_key, + GError **error); + + gboolean (* set_pk) (OstreeSign *self, + GVariant *public_key, + GError **error); + + gboolean (* load_pk) (OstreeSign *self, + gchar *remote_name, + GError **error); + +}; + +_OSTREE_PUBLIC +const gchar * ostree_sign_get_name (OstreeSign *self); + +_OSTREE_PUBLIC +gboolean ostree_sign_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); + + +_OSTREE_PUBLIC +gchar * ostree_sign_metadata_key (OstreeSign *self); + +_OSTREE_PUBLIC +gchar * ostree_sign_metadata_format (OstreeSign *self); + +_OSTREE_PUBLIC +GVariant * ostree_sign_detached_metadata_append (OstreeSign *self, + GVariant *existing_metadata, + GBytes *signature_bytes); + +_OSTREE_PUBLIC +gboolean ostree_sign_commit (OstreeSign *self, + OstreeRepo *repo, + const gchar *commit_checksum, + GCancellable *cancellable, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sign_metadata_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sign_commit_verify (OstreeSign *self, + OstreeRepo *repo, + const gchar *commit_checksum, + GCancellable *cancellable, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sign_set_sk (OstreeSign *self, + GVariant *secret_key, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sign_set_pk (OstreeSign *self, + GVariant *public_key, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_sign_load_pk (OstreeSign *self, + gchar *remote_name, + GError **error); + + +/** + * ostree_sign_list_names: + * + * Return the array with all available sign modules names. + * + * Returns: (transfer full): an array of strings, free when you used it + */ +_OSTREE_PUBLIC +GStrv ostree_sign_list_names(void); + +/** + * ostree_sign_get_by_name: + * + * Tries to find and return proper signing engine by it's name. + * + * Returns: (transfer full): a constant, free when you used it + */ +_OSTREE_PUBLIC +OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error); + +G_END_DECLS + From c3608aa56a4e1551240b7effe2c40bb609211b3d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 31 Jul 2019 01:13:48 +0300 Subject: [PATCH 013/177] sign: add new builtin for signing This builtin allows to sign and verify commit with new signature mechanism. At the moment it is possible to use 'dummy' and 'ed25519' signing modules. 'dummy' module use any ASCII string from command line as a key for commit's signing or verification. Support of ed25519 signature is implemented with `libsoium` library. Secret and public key should be provided in hex presentation via command line. Based on 'gpg-sign' source. Signed-off-by: Denis Pynkin --- Makefile-ostree.am | 1 + src/ostree/main.c | 3 + src/ostree/ot-builtin-sign.c | 211 +++++++++++++++++++++++++++++++++++ src/ostree/ot-builtins.h | 1 + 4 files changed, 216 insertions(+) create mode 100644 src/ostree/ot-builtin-sign.c diff --git a/Makefile-ostree.am b/Makefile-ostree.am index f37d974a..e5767641 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -43,6 +43,7 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-remote.c \ src/ostree/ot-builtin-reset.c \ src/ostree/ot-builtin-rev-parse.c \ + src/ostree/ot-builtin-sign.c \ src/ostree/ot-builtin-summary.c \ src/ostree/ot-builtin-show.c \ src/ostree/ot-builtin-static-delta.c \ diff --git a/src/ostree/main.c b/src/ostree/main.c index a523ff9a..a9f57392 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -109,6 +109,9 @@ static OstreeCommand commands[] = { { "rev-parse", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_rev_parse, "Output the target of a rev" }, + { "sign", OSTREE_BUILTIN_FLAG_NONE, + ostree_builtin_sign, + "Sign a commit" }, { "show", OSTREE_BUILTIN_FLAG_NONE, ostree_builtin_show, "Output a metadata object" }, diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c new file mode 100644 index 00000000..8edd5490 --- /dev/null +++ b/src/ostree/ot-builtin-sign.c @@ -0,0 +1,211 @@ +/* vim:set et sw=2 cin cino=t0,f0,(0,{s,>2s,n-s,^-s,e2s: */ + +/* + * Copyright (C) 2015 Colin Walters + * Copyright (C) 2019 Denis Pynkin (d4s) + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + +#include "ot-main.h" +#include "ot-builtins.h" +#include "ostree.h" +#include "otutil.h" +#include "ostree-core-private.h" +#include "ostree-sign.h" +#include "ostree-sign-dummy.h" +#if defined(HAVE_LIBSODIUM) +#include "ostree-sign-ed25519.h" +#include +#endif + +static gboolean opt_delete; +static gboolean opt_verify; +static char *opt_sign_name; + +/* ATTENTION: + * Please remember to update the bash-completion script (bash/ostree) and + * man page (man/ostree-sign.xml) when changing the option list. + */ + +static GOptionEntry options[] = { + { "delete", 'd', 0, G_OPTION_ARG_NONE, &opt_delete, "Delete signatures having any of the KEY-IDs", NULL}, + { "verify", 0, 0, G_OPTION_ARG_NONE, &opt_verify, "Verify signatures", NULL}, + { "sign-type", 's', 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, +#if defined(HAVE_LIBSODIUM) +#endif + { NULL } +}; + +static void +usage_error (GOptionContext *context, const char *message, GError **error) +{ + g_autofree char *help = g_option_context_get_help (context, TRUE, NULL); + g_printerr ("%s", help); + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, message); +} + +gboolean +ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(GOptionContext) context = NULL; + g_autoptr(OstreeRepo) repo = NULL; + g_autoptr (OstreeSign) sign = NULL; + g_autofree char *resolved_commit = NULL; + const char *commit; + char **key_ids; + int n_key_ids, ii; + gboolean ret = FALSE; +#if defined(HAVE_LIBSODIUM) + g_autoptr (GVariant) ed25519_sk = NULL; + g_autoptr (GVariant) ed25519_pk = NULL; +#endif + + + context = g_option_context_new ("COMMIT KEY-ID..."); + + + if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error)) + goto out; + + if (argc < 2) + { + usage_error (context, "Need a COMMIT to sign or verify", error); + goto out; + } + + commit = argv[1]; + + if (!opt_verify && argc < 3) + { + usage_error (context, "Need at least one KEY-ID to sign with", error); + goto out; + } + + key_ids = argv + 2; + n_key_ids = argc - 2; + + if (!ostree_repo_resolve_rev (repo, commit, FALSE, &resolved_commit, error)) + goto out; + + /* Initialize crypto system */ + if (!opt_sign_name) + opt_sign_name = "ed25519"; + + sign = ostree_sign_get_by_name (opt_sign_name, error); + if (sign == NULL) + { + ret = FALSE; + goto out; + } + + for (ii = 0; ii < n_key_ids; ii++) + { + g_autoptr (GVariant) sk = NULL; + g_autoptr (GVariant) pk = NULL; + g_autofree guchar *key = NULL; + + if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) + { + // Just use the string as signature + sk = g_variant_new_string(key_ids[ii]); + pk = g_variant_new_string(key_ids[ii]); + } + if (opt_verify) + { +#if defined(HAVE_LIBSODIUM) + if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) + { + gsize key_len = 0; + key = g_malloc0 (crypto_sign_PUBLICKEYBYTES); + if (sodium_hex2bin (key, crypto_sign_PUBLICKEYBYTES, + key_ids[ii], strlen (key_ids[ii]), + NULL, &key_len, NULL) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid KEY '%s'", key_ids[ii]); + + goto out; + } + + pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } +#endif + + if (!ostree_sign_set_pk (sign, pk, error)) + { + ret = FALSE; + goto out; + } + + if (ostree_sign_commit_verify (sign, + repo, + resolved_commit, + cancellable, + error)) + ret = TRUE; + } + else + { +#if defined(HAVE_LIBSODIUM) + if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) + { + gsize key_len = 0; + key = g_malloc0 (crypto_sign_SECRETKEYBYTES); + if (sodium_hex2bin (key, crypto_sign_SECRETKEYBYTES, + key_ids[ii], strlen (key_ids[ii]), + NULL, &key_len, NULL) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid KEY '%s'", key_ids[ii]); + + goto out; + } + + sk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } +#endif + if (!ostree_sign_set_sk (sign, sk, error)) + { + ret = FALSE; + goto out; + } + + ret = ostree_sign_commit (sign, + repo, + resolved_commit, + cancellable, + error); + if (ret != TRUE) + goto out; + } + } + + // No valid signature found + if (opt_verify && (ret != TRUE)) + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "No valid signatures found"); + +out: + return ret; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index 12a99b45..e372d359 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -53,6 +53,7 @@ BUILTINPROTO(prune); BUILTINPROTO(refs); BUILTINPROTO(reset); BUILTINPROTO(fsck); +BUILTINPROTO(sign); BUILTINPROTO(show); BUILTINPROTO(static_delta); BUILTINPROTO(summary); From e133cb7b74aa00a13566616d63c223be8292fa4a Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 31 Jul 2019 04:00:19 +0300 Subject: [PATCH 014/177] sign: allow to sign commits from CLI Add signing ability to commit builtin. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-commit.c | 68 ++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 4cca56d0..4bbde92e 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -31,6 +31,12 @@ #include "parse-datetime.h" #include "ostree-repo-private.h" #include "ostree-libarchive-private.h" +#include "ostree-sign.h" +#include "ostree-sign-dummy.h" +#if defined(HAVE_LIBSODIUM) +#include "ostree-sign-ed25519.h" +#include +#endif static char *opt_subject; static char *opt_body; @@ -62,9 +68,11 @@ static gint opt_owner_uid = -1; static gint opt_owner_gid = -1; static gboolean opt_table_output; #ifndef OSTREE_DISABLE_GPGME -static char **opt_key_ids; +static char **opt_gpg_key_ids; static char *opt_gpg_homedir; #endif +static char **opt_key_ids; +static char *opt_sign_name; static gboolean opt_generate_sizes; static gboolean opt_disable_fsync; static char *opt_timestamp; @@ -119,9 +127,11 @@ static GOptionEntry options[] = { { "consume", 0, 0, G_OPTION_ARG_NONE, &opt_consume, "Consume (delete) content after commit (for local directories)", NULL }, { "table-output", 0, 0, G_OPTION_ARG_NONE, &opt_table_output, "Output more information in a KEY: VALUE format", NULL }, #ifndef OSTREE_DISABLE_GPGME - { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"}, + { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_gpg_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, #endif + { "sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "Sign the commit with", "KEY_ID"}, + { "sign-type", 0, 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, { "generate-sizes", 0, 0, G_OPTION_ARG_NONE, &opt_generate_sizes, "Generate size information along with commit metadata", NULL }, { "disable-fsync", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "fsync", 0, 0, G_OPTION_ARG_CALLBACK, parse_fsync_cb, "Specify how to invoke fsync()", "POLICY" }, @@ -419,6 +429,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio OstreeRepoTransactionStats stats; struct CommitFilterData filter_data = { 0, }; g_autofree char *commit_body = NULL; + g_autoptr (OstreeSign) sign = NULL; context = g_option_context_new ("[PATH]"); @@ -832,12 +843,63 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio goto out; } -#ifndef OSTREE_DISABLE_GPGME if (opt_key_ids) { + /* Initialize crypto system */ + if (!opt_sign_name) + opt_sign_name = "ed25519"; + + sign = ostree_sign_get_by_name (opt_sign_name, error); + if (sign == NULL) + goto out; + char **iter; for (iter = opt_key_ids; iter && *iter; iter++) + { + const char *keyid = *iter; + g_autoptr (GVariant) secret_key = NULL; + + if (!g_strcmp0(ostree_sign_get_name (sign), "dummy")) + { + secret_key = g_variant_new_string (keyid); + } +#if defined(HAVE_LIBSODIUM) + else if (!g_strcmp0 (ostree_sign_get_name (sign), "ed25519")) + { + gsize key_len = 0; + key = g_malloc0 (crypto_sign_SECRETKEYBYTES); + if (sodium_hex2bin (key, crypto_sign_SECRETKEYBYTES, + keyid, strlen (keyid), + NULL, &key_len, NULL) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid KEY '%s'", keyid); + + goto out; + } + + secret_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } +#endif + if (!ostree_sign_set_sk (sign, secret_key, error)) + goto out; + + if (!ostree_sign_commit (sign, + repo, + commit_checksum, + cancellable, + error)) + goto out; + } + } + +#ifndef OSTREE_DISABLE_GPGME + if (opt_gpg_key_ids) + { + char **iter; + + for (iter = opt_gpg_key_ids; iter && *iter; iter++) { const char *keyid = *iter; From c09df184544ccd1921855abd072dcf186712376f Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 2 Aug 2019 02:16:56 +0300 Subject: [PATCH 015/177] lib/sign: enable verification for pulling Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 92 ++++++++++++++++++++++++++--- src/libostree/ostree-sign-dummy.c | 1 - src/libostree/ostree-sign-ed25519.c | 1 - 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 381cce47..507bcc2e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -54,6 +54,8 @@ #include #endif +#include "ostree-sign.h" + #define OSTREE_MESSAGE_FETCH_COMPLETE_ID SD_ID128_MAKE(75,ba,3d,eb,0a,f0,41,a9,a4,62,72,ff,85,d9,e7,3e) #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) @@ -105,6 +107,7 @@ typedef struct { gboolean gpg_verify; gboolean gpg_verify_summary; + gboolean sign_verify; gboolean require_static_deltas; gboolean disable_static_deltas; gboolean has_tombstone_commits; @@ -1500,6 +1503,38 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ + if (pull_data->sign_verify) + { + gboolean ret = FALSE; + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); + /* list all signature types in detached metadata and check if signed by any? */ + GStrv names = ostree_sign_list_names(); + for (guint i=0; i < g_strv_length (names); i++) + { + g_autoptr (OstreeSign) sign = ostree_sign_get_by_name (names[i], error); + g_autoptr(GVariant) signatures = NULL; + g_autofree gchar *signature_key = ostree_sign_metadata_key (sign); + g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format (sign); + + signatures = g_variant_lookup_value (detached_metadata, + signature_key, + signature_format); + + /* Set return to true if any sign fit */ + if (!signatures) + continue; + + if (ostree_sign_metadata_verify (sign, + signed_data, + signatures, + error + )) + ret = TRUE; + } + g_strfreev(names); + return ret; + } + return TRUE; } @@ -1829,6 +1864,28 @@ scan_commit_object (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ + if (pull_data->sign_verify && + !g_hash_table_contains (pull_data->verified_commits, checksum)) + { + gboolean ret = FALSE; + /* list all signature types in detached metadata and check if signed by any? */ + GStrv names = ostree_sign_list_names(); + for (guint i=0; i < g_strv_length (names); i++) + { + g_autoptr (OstreeSign) sign = ostree_sign_get_by_name (names[i], error); + + if (ostree_sign_commit_verify (sign, + pull_data->repo, + checksum, + cancellable, + error)) + ret = TRUE; + } + g_strfreev(names); + if (ret == FALSE) + return FALSE; + } + /* If we found a legacy transaction flag, assume we have to scan. * We always do a scan of dirtree objects; see * https://github.com/ostreedev/ostree/issues/543 @@ -3576,6 +3633,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GSource) update_timeout = NULL; gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; + gboolean opt_sign_verify_set = FALSE; gboolean opt_collection_refs_set = FALSE; gboolean opt_n_network_retries_set = FALSE; gboolean opt_ref_keyring_map_set = FALSE; @@ -3610,6 +3668,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_variant_lookup (options, "gpg-verify", "b", &pull_data->gpg_verify); opt_gpg_verify_summary_set = g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary); + opt_sign_verify_set = + g_variant_lookup (options, "sign-verify", "b", &pull_data->sign_verify); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &pull_data->disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas); @@ -3759,7 +3819,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, /* For compatibility with pull-local, don't gpg verify local * pulls by default. */ - if ((pull_data->gpg_verify || pull_data->gpg_verify_summary) && + if ((pull_data->gpg_verify || + pull_data->gpg_verify_summary || + pull_data->sign_verify + ) && pull_data->remote_name == NULL) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -3788,6 +3851,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; #endif /* OSTREE_DISABLE_GPGME */ + /* TODO: read option for remote. */ + if (!opt_sign_verify_set) + opt_sign_verify_set = TRUE; + /* NOTE: If changing this, see the matching implementation in * ostree-sysroot-upgrader.c */ @@ -4647,23 +4714,28 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_string_append_printf (msg, "libostree pull from '%s' for %u refs complete", pull_data->remote_name, g_hash_table_size (requested_refs_to_fetch)); - const char *verify_state; + const char *gpg_verify_state; #ifndef OSTREE_DISABLE_GPGME if (pull_data->gpg_verify_summary) { if (pull_data->gpg_verify) - verify_state = "summary+commit"; + gpg_verify_state = "summary+commit"; else - verify_state = "summary-only"; + gpg_verify_state = "summary-only"; } else - verify_state = (pull_data->gpg_verify ? "commit" : "disabled"); - g_string_append_printf (msg, "\nsecurity: GPG: %s ", verify_state); + gpg_verify_state = (pull_data->gpg_verify ? "commit" : "disabled"); + + g_string_append_printf (msg, "\nsecurity: GPG: %s ", gpg_verify_state); #else - verify_state = "disabled"; - g_string_append_printf (msg, "\nsecurity: %s ", verify_state); + gpg_verify_state = "disabled"; + g_string_append_printf (msg, "\nsecurity: %s ", gpg_verify_state); #endif /* OSTREE_DISABLE_GPGME */ + const char *sign_verify_state; + sign_verify_state = (pull_data->sign_verify ? "commit" : "disabled"); + g_string_append_printf (msg, "\nsecurity: SIGN: %s ", sign_verify_state); + OstreeFetcherURI *first_uri = pull_data->meta_mirrorlist->pdata[0]; g_autofree char *first_scheme = _ostree_fetcher_uri_get_scheme (first_uri); if (g_str_has_prefix (first_scheme, "http")) @@ -4698,7 +4770,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, ot_journal_send ("MESSAGE=%s", msg->str, "MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_MESSAGE_FETCH_COMPLETE_ID), "OSTREE_REMOTE=%s", pull_data->remote_name, - "OSTREE_GPG=%s", verify_state, + "OSTREE_SIGN=%s", sign_verify_state, + "OSTREE_GPG=%s", gpg_verify_state, "OSTREE_SECONDS=%u", n_seconds, "OSTREE_XFER_SIZE=%s", formatted_xferred, NULL); @@ -6023,6 +6096,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, g_variant_dict_insert (&local_options_dict, "gpg-verify", "b", FALSE); #endif /* OSTREE_DISABLE_GPGME */ g_variant_dict_insert (&local_options_dict, "gpg-verify-summary", "b", FALSE); + g_variant_dict_insert (&local_options_dict, "sign-verify", "b", FALSE); g_variant_dict_insert (&local_options_dict, "inherit-transaction", "b", TRUE); if (result->remote->refspec_name != NULL) g_variant_dict_insert (&local_options_dict, "override-remote-name", "s", result->remote->refspec_name); diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index e489a988..4baf656c 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -64,7 +64,6 @@ static void ostree_sign_dummy_class_init (OstreeSignDummyClass *self) { g_debug ("%s enter", __FUNCTION__); - GObjectClass *object_class = G_OBJECT_CLASS(self); } static void diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index e3ab6b57..6a110104 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -67,7 +67,6 @@ static void ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) { g_debug ("%s enter", __FUNCTION__); - GObjectClass *object_class = G_OBJECT_CLASS(self); } static void From 9e8f0f4ca0350cd43c202fe25b99900450e53406 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 2 Aug 2019 02:20:33 +0300 Subject: [PATCH 016/177] tests: add test for commits sign/verification Add tests checking: - sign mechanism is in working state - module 'dummy' is able to sign/verify commit - module 'ed25519' is able to sign/verify commit - both modules could be used for the same commit - 'ostree sign' builtin works with commits - 'ostree commit' builtin able to sign commits Signed-off-by: Denis Pynkin --- Makefile-tests.am | 1 + tests/libtest.sh | 10 ++++ tests/test-signed-commit.sh | 106 ++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100755 tests/test-signed-commit.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 83b0f1a2..8e233466 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -137,6 +137,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-summary-collections.sh \ tests/test-pull-collections.sh \ tests/test-config.sh \ + tests/test-signed-commit.sh \ $(NULL) if USE_GPGME diff --git a/tests/libtest.sh b/tests/libtest.sh index c82bf487..58a9fd9b 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -673,6 +673,16 @@ which_gpg () { echo ${gpg} } +has_libsodium () { + local ret + ${CMD_PREFIX} ostree --version > version.txt + grep -q -e '- libsodium' version.txt + ret=$? + rm -f version.txt + return ${ret} +} + + libtest_cleanup_gpg () { local gpg_homedir=${1:-${test_tmpdir}/gpghome} gpg-connect-agent --homedir "${gpg_homedir}" killagent /bye || true diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh new file mode 100755 index 00000000..08993c60 --- /dev/null +++ b/tests/test-signed-commit.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# Copyright (C) 2019 Collabora Ltd. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..6" + +mkdir ${test_tmpdir}/repo +ostree_repo_init repo --mode="archive" + +echo "Unsigned commit" > file.txt +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Unsigned commit' +COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" + +# Test `ostree sign` with dummy module first +DUMMYSIGN="dummysign" +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy ${COMMIT} ${DUMMYSIGN} + +# Ensure that detached metadata really contain expected string +EXPECTEDSIGN="$(echo $DUMMYSIGN | hexdump -n 9 -e '8/1 "0x%.2x, " 1/1 " 0x%.2x"')" +${CMD_PREFIX} ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.dummy | grep -q -e "${EXPECTEDSIGN}" +echo "ok Detached dummy signature added" + +# Verify vith sign mechanism +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} +echo "ok dummy signature verified" + +echo "Signed commit with dummy key: ${DUMMYSIGN}" >> file.txt +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Signed with dummy module' --sign=${DUMMYSIGN} --sign-type=dummy +COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} +echo "ok commit with dummy signing" + +# Test ostree sign with 'ed25519' module +# Generate private key in PEM format +PEMFILE="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" +openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" + +# tests below require libsodium support +if has_libsodium; then + # Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html + # Extract the private and public parts from generated key. + PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | hexdump -s 12 -e '16/1 "%.2x"')" + SEED="$(openssl pkey -outform DER -in ${PEMFILE} | hexdump -s 16 -e '16/1 "%.2x"')" + # Secret key is concantination of SEED and PUBLIC + SECRET="${SEED}${PUBLIC}" + + echo "SEED = $SEED" + echo "PUBLIC = $PUBLIC" + + echo "Signed commit with ed25519: ${SECRET}" >> file.txt + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s "Signed with ed25519 module" --sign=${SECRET} --sign-type=ed25519 + COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" + + # Ensure that detached metadata contain signature + ${CMD_PREFIX} ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.ed25519 &>/dev/null + echo "ok Detached ed25519 signature added" + + # Verify vith sign mechanism + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} + echo "ok ed25519 signature verified" + + # Check if we able to use all available modules to sign the same commit + echo "Unsigned commit for multi-sign" >> file.txt + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Unsigned commit' + COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" + # Check if we have no signatures + for mod in "dummy" "ed25519"; do + if ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.${mod}; then + echo "Unexpected signature for ${mod} found" + exit 1 + fi + done + + # Sign with all available modules + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy ${COMMIT} ${DUMMYSIGN} + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 ${COMMIT} ${SECRET} + # and verify + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} + echo "ok multiple signing " +else + echo "ok Detached ed25519 signature # SKIP due libsodium unavailability" + echo "ok ed25519 signature verified # SKIP due libsodium unavailability" + echo "ok multiple signing # SKIP due libsodium unavailability" +fi From 2303202c86c96768abb42728b2e5b2090b9d0198 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 9 Aug 2019 22:07:57 +0300 Subject: [PATCH 017/177] sign: API changes for public keys and CLI keys format API changes: - added function `ostree_sign_add_pk()` for multiple public keys using. - `ostree_sign_set_pk()` now substitutes all previously added keys. - added function `ostree_sign_load_pk()` allowed to load keys from file. - `ostree_sign_ed25519_load_pk()` able to load the raw keys list from file. - use base64 encoded public and private ed25519 keys for CLI and keys file. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-repo-pull.c | 17 +- src/libostree/ostree-sign-dummy.h | 2 - src/libostree/ostree-sign-ed25519.c | 238 +++++++++++++++++++++++++--- src/libostree/ostree-sign-ed25519.h | 9 +- src/libostree/ostree-sign.c | 19 ++- src/libostree/ostree-sign.h | 13 +- src/ostree/ot-builtin-commit.c | 9 +- src/ostree/ot-builtin-sign.c | 14 +- tests/test-signed-commit.sh | 8 +- 11 files changed, 278 insertions(+), 53 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index cfc4a340..806bd1a7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -719,6 +719,7 @@ ostree_sign_get_by_name ostree_sign_get_name ostree_sign_detached_metadata_append ostree_sign_metadata_verify +ostree_sign_add_pk ostree_sign_load_pk ostree_sign_set_pk ostree_sign_set_sk diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 4066f383..8be5a3bf 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -33,6 +33,7 @@ global: ostree_sign_metadata_verify; ostree_sign_load_pk; ostree_sign_set_pk; + ostree_sign_add_pk; ostree_sign_set_sk; ostree_sign_dummy_get_type; ostree_sign_ed25519_get_type; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 507bcc2e..781c2458 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1508,13 +1508,21 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, gboolean ret = FALSE; g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); /* list all signature types in detached metadata and check if signed by any? */ - GStrv names = ostree_sign_list_names(); + g_auto(GStrv) names = ostree_sign_list_names(); for (guint i=0; i < g_strv_length (names); i++) { - g_autoptr (OstreeSign) sign = ostree_sign_get_by_name (names[i], error); + g_autoptr (OstreeSign) sign = NULL; g_autoptr(GVariant) signatures = NULL; - g_autofree gchar *signature_key = ostree_sign_metadata_key (sign); - g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format (sign); + g_autofree gchar *signature_key = NULL; + g_autofree GVariantType *signature_format = NULL; + + if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) + { + g_error_free (*error); + continue; + } + signature_key = ostree_sign_metadata_key (sign); + signature_format = (GVariantType *) ostree_sign_metadata_format (sign); signatures = g_variant_lookup_value (detached_metadata, signature_key, @@ -1531,7 +1539,6 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, )) ret = TRUE; } - g_strfreev(names); return ret; } diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index 8bbd407d..73bad135 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -57,7 +57,5 @@ gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, gboolean ostree_sign_dummy_set_signature (OstreeSign *self, GVariant *key, GError **error); -void ostree_sign_dummy_finalize (GObject *gobject); - G_END_DECLS diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 6a110104..779c0b27 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -30,17 +30,25 @@ #include #endif +#define G_LOG_DOMAIN "OSTreeSign" + #define OSTREE_SIGN_ED25519_NAME "ed25519" #define OSTREE_SIGN_METADATA_ED25519_KEY "ostree.sign.ed25519" #define OSTREE_SIGN_METADATA_ED25519_TYPE "aay" +#if 0 +#define SIGNIFY_COMMENT_HEADER "untrusted comment:" +#define SIGNIFY_ID_LENGTH 8 +#define SIGNIFY_MAGIC_ED25519 "Ed" +#endif + struct _OstreeSignEd25519 { GObject parent; gboolean initialized; guchar *secret_key; - guchar *public_key; + GList *public_keys; }; static void @@ -61,12 +69,32 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) self->metadata_verify = ostree_sign_ed25519_metadata_verify; self->set_sk = ostree_sign_ed25519_set_sk; self->set_pk = ostree_sign_ed25519_set_pk; + self->add_pk = ostree_sign_ed25519_add_pk; + self->load_pk = ostree_sign_ed25519_load_pk; +} + +static void +ostree_sign_ed25519_finalize (GObject *object) +{ + g_debug ("%s enter", __FUNCTION__); +#if 0 + OstreeSignEd25519 *self = OSTREE_SIGN_ED25519 (object); + + if (self->public_keys != NULL) + g_list_free_full (self->public_keys, g_object_unref); + if (self->secret_key != NULL) + free(self->secret_key); +#endif + G_OBJECT_CLASS (ostree_sign_ed25519_parent_class)->finalize (object); } static void ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) { g_debug ("%s enter", __FUNCTION__); + GObjectClass *object_class = G_OBJECT_CLASS (self); + + object_class->finalize = ostree_sign_ed25519_finalize; } static void @@ -76,7 +104,7 @@ ostree_sign_ed25519_init (OstreeSignEd25519 *self) self->initialized = TRUE; self->secret_key = NULL; - self->public_key = NULL; + self->public_keys = NULL; #ifdef HAVE_LIBSODIUM if (sodium_init() < 0) @@ -188,7 +216,7 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, goto err; } - if ((sign->initialized != TRUE) || (sign->public_key == NULL)) + if ((sign->initialized != TRUE) || (sign->public_keys == NULL)) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Not able to verify: libsodium library isn't initialized properly"); @@ -207,20 +235,26 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, g_debug("Read signature %d: %s", (gint)i, g_variant_print(child, TRUE)); - if (crypto_sign_verify_detached ((guchar *) g_variant_get_data (child), - g_bytes_get_data (data, NULL), - g_bytes_get_size (data), - sign->public_key) != 0) + for (GList *public_key = sign->public_keys; + public_key != NULL; + public_key = public_key->next) { - /* Incorrect signature! */ - g_debug("Signature couldn't be verified with key '%s'", - sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, crypto_sign_PUBLICKEYBYTES)); - } - else - { - ret = TRUE; - g_debug ("Signature verified successfully with key '%s'", - sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, crypto_sign_PUBLICKEYBYTES)); + if (crypto_sign_verify_detached ((guchar *) g_variant_get_data (child), + g_bytes_get_data (data, NULL), + g_bytes_get_size (data), + public_key->data) != 0) + { + /* Incorrect signature! */ + g_debug("Signature couldn't be verified with key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + } + else + { + ret = TRUE; + g_debug ("Signature verified successfully with key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + break; + } } } @@ -297,7 +331,7 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, } hex = g_malloc0 (crypto_sign_SECRETKEYBYTES*2 + 1); - g_debug ("Set ed25519 secret key = %s", sodium_bin2hex (hex, crypto_sign_SECRETKEYBYTES*2+1, sign->secret_key, n_elements)); +// g_debug ("Set ed25519 secret key = %s", sodium_bin2hex (hex, crypto_sign_SECRETKEYBYTES*2+1, sign->secret_key, n_elements)); return TRUE; @@ -313,16 +347,35 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + /* Substitute the key(s) with a new one */ + if (sign->public_keys != NULL) + { + g_list_free_full (sign->public_keys, g_object_unref); + sign->public_keys = NULL; + } + + return ostree_sign_ed25519_add_pk (self, public_key, error); +} + +gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, + GVariant *public_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); g_autofree char * hex = NULL; + gpointer key = NULL; gsize n_elements = 0; - g_free (sign->public_key); - sign->public_key = (guchar *) g_variant_get_fixed_array (public_key, &n_elements, sizeof(guchar)); + key = (gpointer) g_variant_get_fixed_array (public_key, &n_elements, sizeof(guchar)); hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); - g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, n_elements)); + g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); if (n_elements != crypto_sign_PUBLICKEYBYTES) { @@ -331,7 +384,9 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, goto err; } - g_debug ("Set ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, sign->public_key, n_elements)); + key = g_memdup (key, n_elements); + if (g_list_find (sign->public_keys, key) == NULL) + sign->public_keys = g_list_prepend (sign->public_keys, key); return TRUE; @@ -339,3 +394,144 @@ err: #endif /* HAVE_LIBSODIUM */ return FALSE; } + + +static gboolean +load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError **error) +{ + g_return_val_if_fail (key_data_in, FALSE); +#ifdef HAVE_LIBSODIUM + gboolean ret = FALSE; + +#if 0 +/* Try to load the public key in signify format from the stream + * https://www.openbsd.org/papers/bsdcan-signify.html + * + * FIXME: Not sure if we need to support that format. + * */ + g_autofree gchar * comment = NULL; + while (TRUE) + { + gsize len = 0; + g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); + if (error) + goto err; + + if (line) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Signify format for ed25519 public key not found"); + goto err; + } + + if (comment == NULL) + { + /* Scan for the comment first and compare with prefix&suffix */ + if (g_str_has_prefix (line, SIGNIFY_COMMENT_HEADER) && g_str_has_suffix (line, "public key")) + /* Save comment without the prefix and blank space */ + comment = g_strdup (line + strlen(SIGNIFY_COMMENT_HEADER) + 1); + } + else + { + /* Read the key itself */ + /* base64 encoded key */ + gsize keylen = 0; + g_autofree guchar *key = g_base64_decode (line, &keylen); + + /* Malformed key */ + if (keylen != SIGNIFY_ID_LENGTH || + strncmp (line, SIGNIFY_MAGIC_ED25519, strlen(SIGNIFY_MAGIC_ED25519)) != 0) + continue; + + } + } +#endif /* 0 */ + + /* Use simple file format with just a list of base64 public keys per line */ + while (TRUE) + { + gsize len = 0; + g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); + g_autoptr (GVariant) pk = NULL; + + if (*error != NULL) + goto err; + + if (line == NULL) + goto out; + + /* Read the key itself */ + /* base64 encoded key */ + gsize key_len = 0; + g_autofree guchar *key = g_base64_decode (line, &key_len); + + pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + if (ostree_sign_ed25519_add_pk (self, pk, error)) + { + ret = TRUE; + g_debug ("Added public key: %s", line); + } + else + g_debug ("Invalid public key: %s", line); + } + +out: + return ret; + +err: +#endif /* HAVE_LIBSODIUM */ + return FALSE; +} + +gboolean +ostree_sign_ed25519_load_pk (OstreeSign *self, + GVariant *options, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + g_autoptr (GFile) keyfile = NULL; + g_autoptr (GFileInputStream) key_stream_in = NULL; + g_autoptr (GDataInputStream) key_data_in = NULL; + + const gchar *remote_name = NULL; + const gchar *filename = NULL; + + /* Clear already loaded keys */ + if (sign->public_keys != NULL) + { + g_list_free_full (sign->public_keys, g_object_unref); + sign->public_keys = NULL; + } + + /* Check if the name of remote is provided */ + if (! g_variant_lookup (options, "remote", "&s", &remote_name)) + remote_name = OSTREE_SIGN_ALL_REMOTES; + + /* Read filename or use will-known if not provided */ + if (! g_variant_lookup (options, "filename", "&s", &filename)) + { + // TODO: define well-known places and load file(s) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Please provide a filename to load"); + goto err; + } + + keyfile = g_file_new_for_path (filename); + key_stream_in = g_file_read (keyfile, NULL, error); + if (key_stream_in == NULL) + goto err; + + key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); + g_assert (key_data_in != NULL); + + if (!load_pk_from_stream (self, key_data_in, error)) + goto err; + + return TRUE; +err: + return FALSE; +} + diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index d4a6b56d..797ac138 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -63,7 +63,13 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, GVariant *public_key, GError **error); -void ostree_sign_ed25519_finalize (GObject *gobject); +gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, + GVariant *public_key, + GError **error); + +gboolean ostree_sign_ed25519_load_pk (OstreeSign *self, + GVariant *options, + GError **error); _OSTREE_PUBLIC gboolean ostree_sign_ed25519_keypair_generate (OstreeSign *self, @@ -71,5 +77,6 @@ gboolean ostree_sign_ed25519_keypair_generate (OstreeSign *self, GVariant **out_public_key, GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 96455f86..0708395c 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -88,22 +88,31 @@ gboolean ostree_sign_set_pk (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->set_pk (self, public_key, error); } +gboolean ostree_sign_add_pk (OstreeSign *self, + GVariant *public_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + if (OSTREE_SIGN_GET_IFACE (self)->add_pk == NULL) + return TRUE; + + return OSTREE_SIGN_GET_IFACE (self)->add_pk (self, public_key, error); +} + /* Load private keys for verification from anywhere. * No need to have the same function for secret keys -- the signing SW must do it in it's own way * */ gboolean ostree_sign_load_pk (OstreeSign *self, - gchar *remote_name, + GVariant *options, GError **error) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->load_pk != NULL, FALSE); - if (remote_name == NULL) - remote_name = OSTREE_SIGN_ALL_REMOTES; - - return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, remote_name, error); + return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, options, error); } gboolean ostree_sign_data (OstreeSign *self, diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index f06206aa..78e2487d 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -68,8 +68,12 @@ struct _OstreeSignInterface GVariant *public_key, GError **error); + gboolean (* add_pk) (OstreeSign *self, + GVariant *public_key, + GError **error); + gboolean (* load_pk) (OstreeSign *self, - gchar *remote_name, + GVariant *options, GError **error); }; @@ -126,9 +130,14 @@ gboolean ostree_sign_set_pk (OstreeSign *self, GVariant *public_key, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sign_add_pk (OstreeSign *self, + GVariant *public_key, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sign_load_pk (OstreeSign *self, - gchar *remote_name, + GVariant *options, GError **error); diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 4bbde92e..89ada19e 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -868,11 +868,10 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio else if (!g_strcmp0 (ostree_sign_get_name (sign), "ed25519")) { gsize key_len = 0; - key = g_malloc0 (crypto_sign_SECRETKEYBYTES); - if (sodium_hex2bin (key, crypto_sign_SECRETKEYBYTES, - keyid, strlen (keyid), - NULL, &key_len, NULL) != 0) - { + g_autofree guchar *key = g_base64_decode (keyid, &key_len); + + if ( key_len != crypto_sign_SECRETKEYBYTES) + { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid KEY '%s'", keyid); diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index 8edd5490..e36a50f1 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -136,10 +136,9 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) { gsize key_len = 0; - key = g_malloc0 (crypto_sign_PUBLICKEYBYTES); - if (sodium_hex2bin (key, crypto_sign_PUBLICKEYBYTES, - key_ids[ii], strlen (key_ids[ii]), - NULL, &key_len, NULL) != 0) + g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); + + if ( key_len != crypto_sign_PUBLICKEYBYTES) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid KEY '%s'", key_ids[ii]); @@ -170,10 +169,9 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) { gsize key_len = 0; - key = g_malloc0 (crypto_sign_SECRETKEYBYTES); - if (sodium_hex2bin (key, crypto_sign_SECRETKEYBYTES, - key_ids[ii], strlen (key_ids[ii]), - NULL, &key_len, NULL) != 0) + g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); + + if ( key_len != crypto_sign_SECRETKEYBYTES) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid KEY '%s'", key_ids[ii]); diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 08993c60..55945f8a 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -60,16 +60,16 @@ openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" if has_libsodium; then # Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html # Extract the private and public parts from generated key. - PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | hexdump -s 12 -e '16/1 "%.2x"')" - SEED="$(openssl pkey -outform DER -in ${PEMFILE} | hexdump -s 16 -e '16/1 "%.2x"')" + PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" + SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" # Secret key is concantination of SEED and PUBLIC - SECRET="${SEED}${PUBLIC}" + SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" echo "SEED = $SEED" echo "PUBLIC = $PUBLIC" echo "Signed commit with ed25519: ${SECRET}" >> file.txt - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s "Signed with ed25519 module" --sign=${SECRET} --sign-type=ed25519 + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s "Signed with ed25519 module" --sign="${SECRET}" --sign-type=ed25519 COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" # Ensure that detached metadata contain signature From 06cfcd9a8a2e11460ab58134477ae23657c3f2ea Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 19 Aug 2019 02:47:45 +0300 Subject: [PATCH 018/177] builtin/sign: allow to provide the file with public keys Added option `--keys-file` for `ostree sign`. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-sign.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index e36a50f1..5093e3c6 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -41,6 +41,7 @@ static gboolean opt_delete; static gboolean opt_verify; static char *opt_sign_name; +static char *opt_filename; /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -52,6 +53,7 @@ static GOptionEntry options[] = { { "verify", 0, 0, G_OPTION_ARG_NONE, &opt_verify, "Verify signatures", NULL}, { "sign-type", 's', 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, #if defined(HAVE_LIBSODIUM) + { "keys-file", 's', 0, G_OPTION_ARG_STRING, &opt_filename, "Read public key(s) from file", "NAME"}, #endif { NULL } }; @@ -198,6 +200,32 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } } + /* Read public signatures from file */ + if (opt_verify && opt_filename) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); + g_variant_builder_add (builder, "{sv}", "test", g_variant_new_string (opt_filename)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + { + ret = FALSE; + goto out; + } + if (ostree_sign_commit_verify (sign, + repo, + resolved_commit, + cancellable, + error)) + ret = TRUE; + if (ret != TRUE) + goto out; + } /* Check via file */ + // No valid signature found if (opt_verify && (ret != TRUE)) g_set_error_literal (error, From 0b55db9b2fad6d48216e130386a4d6980a336614 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 19 Aug 2019 02:49:50 +0300 Subject: [PATCH 019/177] tests/sign: check public keys load from file Test ed25519 public keys load from file and verify signed commit against that file. Signed-off-by: Denis Pynkin --- tests/test-signed-commit.sh | 116 +++++++++++++++++++++++------------- 1 file changed, 74 insertions(+), 42 deletions(-) diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 55945f8a..2c547542 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..6" +echo "1..7" mkdir ${test_tmpdir}/repo ostree_repo_init repo --mode="archive" @@ -51,56 +51,88 @@ COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} echo "ok commit with dummy signing" +# tests below require libsodium support +if ! has_libsodium; then + echo "ok Detached ed25519 signature # SKIP due libsodium unavailability" + echo "ok ed25519 signature verified # SKIP due libsodium unavailability" + echo "ok multiple signing # SKIP due libsodium unavailability" + echo "ok verify ed25519 keys file # SKIP due libsodium unavailability" + exit 0 +fi + # Test ostree sign with 'ed25519' module # Generate private key in PEM format PEMFILE="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" -# tests below require libsodium support -if has_libsodium; then - # Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html - # Extract the private and public parts from generated key. - PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" - SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" - # Secret key is concantination of SEED and PUBLIC - SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" +# Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html +# Extract the private and public parts from generated key. +PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" +SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" +# Secret key is concantination of SEED and PUBLIC +SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" - echo "SEED = $SEED" - echo "PUBLIC = $PUBLIC" +echo "SEED = $SEED" +echo "PUBLIC = $PUBLIC" - echo "Signed commit with ed25519: ${SECRET}" >> file.txt - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s "Signed with ed25519 module" --sign="${SECRET}" --sign-type=ed25519 - COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" +echo "Signed commit with ed25519: ${SECRET}" >> file.txt +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s "Signed with ed25519 module" --sign="${SECRET}" --sign-type=ed25519 +COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" - # Ensure that detached metadata contain signature - ${CMD_PREFIX} ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.ed25519 &>/dev/null - echo "ok Detached ed25519 signature added" +# Ensure that detached metadata contain signature +${CMD_PREFIX} ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.ed25519 &>/dev/null +echo "ok Detached ed25519 signature added" - # Verify vith sign mechanism - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} - echo "ok ed25519 signature verified" +# Verify vith sign mechanism +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} +echo "ok ed25519 signature verified" - # Check if we able to use all available modules to sign the same commit - echo "Unsigned commit for multi-sign" >> file.txt - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Unsigned commit' - COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" - # Check if we have no signatures - for mod in "dummy" "ed25519"; do - if ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.${mod}; then - echo "Unexpected signature for ${mod} found" - exit 1 - fi - done +# Check if we able to use all available modules to sign the same commit +echo "Unsigned commit for multi-sign" >> file.txt +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Unsigned commit' +COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" +# Check if we have no signatures +for mod in "dummy" "ed25519"; do + if ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=ostree.sign.${mod}; then + echo "Unexpected signature for ${mod} found" + exit 1 + fi +done - # Sign with all available modules - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy ${COMMIT} ${DUMMYSIGN} - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 ${COMMIT} ${SECRET} - # and verify - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} - ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} - echo "ok multiple signing " -else - echo "ok Detached ed25519 signature # SKIP due libsodium unavailability" - echo "ok ed25519 signature verified # SKIP due libsodium unavailability" - echo "ok multiple signing # SKIP due libsodium unavailability" +# Sign with all available modules +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy ${COMMIT} ${DUMMYSIGN} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 ${COMMIT} ${SECRET} +# and verify +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} +echo "ok multiple signing " + +# Prepare files with public ed25519 signatures +PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" + +# Test if file contain no keys +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then + exit 1 fi +# Test if have a problem with file object +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${test_tmpdir} ${COMMIT}; then + exit 1 +fi +# Test with single key in list +echo ${PUBLIC} > ${PUBKEYS} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} + +# Test the file with multiple keys without a valid public key +for((i=0;i<100;i++)); do + # Generate a list with some public signatures + openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 +done > ${PUBKEYS} +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then + exit 1 +fi + +# Add correct key into the list +echo ${PUBLIC} >> ${PUBKEYS} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} + +echo "ok verify ed25519 keys file" From 4b9232b1fe46cf84ba787336eb0aaa4418e96698 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 20 Aug 2019 00:56:27 +0300 Subject: [PATCH 020/177] builtin/sign: remove libsodium-specific code Use only common sign API without libsoduim parts. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-commit.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 89ada19e..7d412639 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -32,11 +32,6 @@ #include "ostree-repo-private.h" #include "ostree-libarchive-private.h" #include "ostree-sign.h" -#include "ostree-sign-dummy.h" -#if defined(HAVE_LIBSODIUM) -#include "ostree-sign-ed25519.h" -#include -#endif static char *opt_subject; static char *opt_body; @@ -864,23 +859,13 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio { secret_key = g_variant_new_string (keyid); } -#if defined(HAVE_LIBSODIUM) else if (!g_strcmp0 (ostree_sign_get_name (sign), "ed25519")) { gsize key_len = 0; g_autofree guchar *key = g_base64_decode (keyid, &key_len); - if ( key_len != crypto_sign_SECRETKEYBYTES) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid KEY '%s'", keyid); - - goto out; - } - secret_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } -#endif if (!ostree_sign_set_sk (sign, secret_key, error)) goto out; From 2d391266282d7e61476434cfcd4c7569fd1ec11b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 21:47:10 +0300 Subject: [PATCH 021/177] sign: fix unneeded objects creation Do not create objects just for supported modules list. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign.c | 81 ++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 0708395c..5f7078f1 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -38,8 +38,31 @@ #include "ostree-sign-ed25519.h" #endif +#undef G_LOG_DOMAIN #define G_LOG_DOMAIN "OSTreeSign" +typedef struct +{ + gchar *name; + GType type; +} _sign_type; + +_sign_type sign_types[] = +{ +#if defined(HAVE_LIBSODIUM) + {"ed25519", 0}, +#endif + {"dummy", 0} +}; + +enum +{ +#if defined(HAVE_LIBSODIUM) + SIGN_ED25519, +#endif + SIGN_DUMMY +}; + G_DEFINE_INTERFACE (OstreeSign, ostree_sign, G_TYPE_OBJECT) static void @@ -110,7 +133,8 @@ ostree_sign_load_pk (OstreeSign *self, { g_debug ("%s enter", __FUNCTION__); - g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->load_pk != NULL, FALSE); + if (OSTREE_SIGN_GET_IFACE (self)->load_pk == NULL) + return FALSE; return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, options, error); } @@ -208,14 +232,6 @@ ostree_sign_commit_verify (OstreeSign *self, g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit_variant); - /* XXX This is a hackish way to indicate to use ALL remote-specific - * keyrings in the signature verification. We want this when - * verifying a signed commit that's already been pulled. */ -/* - if (remote_name == NULL) - remote_name = OSTREE_ALL_REMOTES; -*/ - g_autoptr(GVariant) signatures = NULL; g_autofree gchar *signature_key = ostree_sign_metadata_key(self); @@ -246,33 +262,31 @@ OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error) { g_debug ("%s enter", __FUNCTION__); - GType types [] = { + OstreeSign *sign = NULL; + + /* Get types if not initialized yet */ #if defined(HAVE_LIBSODIUM) - OSTREE_TYPE_SIGN_ED25519, + if (sign_types[SIGN_ED25519].type == 0) + sign_types[SIGN_ED25519].type = OSTREE_TYPE_SIGN_ED25519; #endif - OSTREE_TYPE_SIGN_DUMMY - }; - OstreeSign *ret = NULL; + if (sign_types[SIGN_DUMMY].type == 0) + sign_types[SIGN_DUMMY].type = OSTREE_TYPE_SIGN_DUMMY; - for (gint i=0; i < G_N_ELEMENTS(types); i++) + for (gint i=0; i < G_N_ELEMENTS(sign_types); i++) { - g_autoptr (OstreeSign) sign = g_object_new (types[i], NULL); - g_autofree gchar *sign_name = OSTREE_SIGN_GET_IFACE (sign)->get_name(sign); - - g_debug ("Found '%s' signing module", sign_name); - - if (g_strcmp0 (name, sign_name) == 0) - { - ret = g_steal_pointer (&sign); - break; - } + if (g_strcmp0 (name, sign_types[i].name) == 0) + { + g_debug ("Found '%s' signing module", sign_types[i].name); + sign = g_object_new (sign_types[i].type, NULL); + break; + } } - if (ret == NULL) + if (sign == NULL) g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Requested signature type is not implemented"); - return ret; + return sign; } @@ -336,19 +350,12 @@ GStrv ostree_sign_list_names(void) { g_debug ("%s enter", __FUNCTION__); - GType types [] = { -#if defined(HAVE_LIBSODIUM) - OSTREE_TYPE_SIGN_ED25519, -#endif - OSTREE_TYPE_SIGN_DUMMY - }; - GStrv names = g_new0 (char *, G_N_ELEMENTS(types)+1); + GStrv names = g_new0 (char *, G_N_ELEMENTS(sign_types) + 1); gint i = 0; - for (i=0; i < G_N_ELEMENTS(types); i++) + for (i=0; i < G_N_ELEMENTS(sign_types); i++) { - g_autoptr (OstreeSign) sign = g_object_new (types[i], NULL); - names[i] = OSTREE_SIGN_GET_IFACE (sign)->get_name(sign); + names[i] = g_strdup(sign_types[i].name); g_debug ("Found '%s' signing module", names[i]); } From a8521a7c3b256c7f9fb55a93f4e038fb89bcc37b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 21:51:03 +0300 Subject: [PATCH 022/177] sign: fix error return for dummy module Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-dummy.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 4baf656c..34660cec 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -27,6 +27,9 @@ #include "ostree-sign-dummy.h" #include +#undef G_LOG_DOMAIN +#define G_LOG_DOMAIN "OSTreeSign" + #define OSTREE_SIGN_DUMMY_NAME "dummy" #define OSTREE_SIGN_METADATA_DUMMY_KEY "ostree.sign.dummy" @@ -174,6 +177,10 @@ gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, if (!g_strcmp0(sign_ascii, sign->signature_ascii)) ret = TRUE; } + if (ret == FALSE && *error == NULL) + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: dummy: incorrect signature"); err: return ret; From fe3a839ae7ddc420853635e2413fb94ca1815777 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 21:55:53 +0300 Subject: [PATCH 023/177] builtin/sign: remove libsodium dependency Now do not need to compile/link builtin with external dependencies. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-sign.c | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index 5093e3c6..6baeb850 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -32,11 +32,6 @@ #include "otutil.h" #include "ostree-core-private.h" #include "ostree-sign.h" -#include "ostree-sign-dummy.h" -#if defined(HAVE_LIBSODIUM) -#include "ostree-sign-ed25519.h" -#include -#endif static gboolean opt_delete; static gboolean opt_verify; @@ -134,23 +129,12 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } if (opt_verify) { -#if defined(HAVE_LIBSODIUM) if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) { gsize key_len = 0; g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); - - if ( key_len != crypto_sign_PUBLICKEYBYTES) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid KEY '%s'", key_ids[ii]); - - goto out; - } - pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } -#endif if (!ostree_sign_set_pk (sign, pk, error)) { @@ -167,23 +151,13 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } else { -#if defined(HAVE_LIBSODIUM) if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) { gsize key_len = 0; g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); - - if ( key_len != crypto_sign_SECRETKEYBYTES) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid KEY '%s'", key_ids[ii]); - - goto out; - } - sk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } -#endif + if (!ostree_sign_set_sk (sign, sk, error)) { ret = FALSE; @@ -208,7 +182,6 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); - g_variant_builder_add (builder, "{sv}", "test", g_variant_new_string (opt_filename)); options = g_variant_builder_end (builder); if (!ostree_sign_load_pk (sign, options, error)) From 2831028c419ed33bf2ac3bf91d7c3b5f2b07194a Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 21:59:34 +0300 Subject: [PATCH 024/177] sign: fixes for ed25519 for loading public keys from files Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 64 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 779c0b27..299c5441 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -30,6 +30,7 @@ #include #endif +#undef G_LOG_DOMAIN #define G_LOG_DOMAIN "OSTreeSign" #define OSTREE_SIGN_ED25519_NAME "ed25519" @@ -73,28 +74,10 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) self->load_pk = ostree_sign_ed25519_load_pk; } -static void -ostree_sign_ed25519_finalize (GObject *object) -{ - g_debug ("%s enter", __FUNCTION__); -#if 0 - OstreeSignEd25519 *self = OSTREE_SIGN_ED25519 (object); - - if (self->public_keys != NULL) - g_list_free_full (self->public_keys, g_object_unref); - if (self->secret_key != NULL) - free(self->secret_key); -#endif - G_OBJECT_CLASS (ostree_sign_ed25519_parent_class)->finalize (object); -} - static void ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) { g_debug ("%s enter", __FUNCTION__); - GObjectClass *object_class = G_OBJECT_CLASS (self); - - object_class->finalize = ostree_sign_ed25519_finalize; } static void @@ -154,7 +137,6 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, goto err; } - g_debug ("sign: data hash = 0x%x", g_bytes_hash(data)); *signature = g_bytes_new (sig, sig_size); return TRUE; #endif /* HAVE_LIBSODIUM */ @@ -189,9 +171,9 @@ gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) } gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error) + GBytes *data, + GVariant *signatures, + GError **error) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -216,7 +198,7 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, goto err; } - if ((sign->initialized != TRUE) || (sign->public_keys == NULL)) + if (sign->initialized != TRUE) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Not able to verify: libsodium library isn't initialized properly"); @@ -224,6 +206,20 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, } #ifdef HAVE_LIBSODIUM + /* If no keys pre-loaded then, + * try to load public keys from storage(s) */ + if (sign->public_keys == NULL) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + options = g_variant_builder_end (builder); + + if (!ostree_sign_ed25519_load_pk (self, options, error)) + goto err; + } + g_debug ("verify: data hash = 0x%x", g_bytes_hash(data)); for (gsize i = 0; i < g_variant_n_children(signatures); i++) @@ -397,7 +393,7 @@ err: static gboolean -load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError **error) +_load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError **error) { g_return_val_if_fail (key_data_in, FALSE); #ifdef HAVE_LIBSODIUM @@ -496,7 +492,6 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, g_autoptr (GFileInputStream) key_stream_in = NULL; g_autoptr (GDataInputStream) key_data_in = NULL; - const gchar *remote_name = NULL; const gchar *filename = NULL; /* Clear already loaded keys */ @@ -506,16 +501,19 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, sign->public_keys = NULL; } - /* Check if the name of remote is provided */ - if (! g_variant_lookup (options, "remote", "&s", &remote_name)) - remote_name = OSTREE_SIGN_ALL_REMOTES; - /* Read filename or use will-known if not provided */ if (! g_variant_lookup (options, "filename", "&s", &filename)) { - // TODO: define well-known places and load file(s) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Please provide a filename to load"); + /* TODO: define well-known places to read */ + /* TODO: scan directories */ + filename = "/etc/ostree/trusted.ed25519"; + } + + if (!g_file_test (filename, G_FILE_TEST_IS_REGULAR)) + { + g_debug ("Can't open file '%s' with pulic keys", filename); + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "File object '%s' is not a regular file", filename); goto err; } @@ -527,7 +525,7 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); g_assert (key_data_in != NULL); - if (!load_pk_from_stream (self, key_data_in, error)) + if (!_load_pk_from_stream (self, key_data_in, error)) goto err; return TRUE; From 3386893debd893bc2e82eaff437b5e16c22fc002 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 22:08:10 +0300 Subject: [PATCH 025/177] sign: check signatures for pulled commits If `verification-key` is set for remote it is used as a public key for checking the commit pulled from that remote. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 114 ++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 11 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 781c2458..b16b668d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1505,6 +1505,17 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, if (pull_data->sign_verify) { + /* Nothing to check if detached metadata is absent */ + if (detached_metadata == NULL) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't verify commit without detached metadata"); + return FALSE; + } + /* Shouldn't happen, but see comment in process_verify_result() */ + if (g_hash_table_contains (pull_data->verified_commits, checksum)) + return TRUE; + gboolean ret = FALSE; g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); /* list all signature types in detached metadata and check if signed by any? */ @@ -1512,13 +1523,14 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, for (guint i=0; i < g_strv_length (names); i++) { g_autoptr (OstreeSign) sign = NULL; - g_autoptr(GVariant) signatures = NULL; + g_autoptr (GVariant) signatures = NULL; g_autofree gchar *signature_key = NULL; g_autofree GVariantType *signature_format = NULL; + g_autofree gchar *pk_ascii = NULL; if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) { - g_error_free (*error); + g_clear_error (error); continue; } signature_key = ostree_sign_metadata_key (sign); @@ -1528,17 +1540,55 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, signature_key, signature_format); - /* Set return to true if any sign fit */ if (!signatures) continue; + /* TODO: load keys for remote here */ + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-key", NULL, + &pk_ascii, NULL); + if (pk_ascii != NULL) + { + g_autoptr (GVariant) pk = NULL; + + if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) + { + // Just use the string as signature + pk = g_variant_new_string(pk_ascii); + } + else if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) + { + gsize key_len = 0; + g_autofree guchar *key = g_base64_decode (pk_ascii, &key_len); + pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } + + if (!ostree_sign_set_pk (sign, pk, error)) + g_clear_error (error); + } + + /* Set return to true if any sign fit */ if (ostree_sign_metadata_verify (sign, signed_data, signatures, error )) ret = TRUE; + else + g_clear_error (error); } + + /* Mark the commit as verified to avoid double verification + * see process_verify_result () for rationale */ + if (ret) + { + g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); + } + else + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't verify commit"); + return ret; } @@ -1876,21 +1926,60 @@ scan_commit_object (OtPullData *pull_data, { gboolean ret = FALSE; /* list all signature types in detached metadata and check if signed by any? */ - GStrv names = ostree_sign_list_names(); + g_auto (GStrv) names = ostree_sign_list_names(); for (guint i=0; i < g_strv_length (names); i++) { - g_autoptr (OstreeSign) sign = ostree_sign_get_by_name (names[i], error); + g_autoptr (OstreeSign) sign = NULL; + g_autofree gchar *pk_ascii = NULL; + if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) + { + g_clear_error (error); + continue; + } + /* TODO: load keys for remote here */ + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-key", NULL, + &pk_ascii, NULL); + if (pk_ascii != NULL) + { + g_autoptr (GVariant) pk = NULL; + + if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) + { + // Just use the string as signature + pk = g_variant_new_string(pk_ascii); + } + else if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) + { + gsize key_len = 0; + g_autofree guchar *key = g_base64_decode (pk_ascii, &key_len); + pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } + + if (!ostree_sign_set_pk (sign, pk, error)) + g_clear_error (error); + } + + + /* Set return to true if any sign fit */ if (ostree_sign_commit_verify (sign, pull_data->repo, checksum, cancellable, error)) ret = TRUE; + else + g_clear_error (error); + + } + if (!ret) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't verify commit"); + return FALSE; } - g_strfreev(names); - if (ret == FALSE) - return FALSE; } /* If we found a legacy transaction flag, assume we have to scan. @@ -3857,10 +3946,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, &pull_data->gpg_verify_summary, error)) goto out; #endif /* OSTREE_DISABLE_GPGME */ - - /* TODO: read option for remote. */ + /* Fetch verification settings from remote if it wasn't already + * explicitly set in the options. */ if (!opt_sign_verify_set) - opt_sign_verify_set = TRUE; + if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, + "sign-verify", TRUE, + &pull_data->sign_verify, error)) + goto out; /* NOTE: If changing this, see the matching implementation in * ostree-sysroot-upgrader.c From 82c773710cc843e04f461df982a42a4f24f1b0de Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 26 Aug 2019 22:11:32 +0300 Subject: [PATCH 026/177] tests/sign: add initial test for pulling Test if we pull signed commits from remote. Signed-off-by: Denis Pynkin --- Makefile-tests.am | 6 +++ tests/test-signed-pull.sh | 92 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100755 tests/test-signed-pull.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 8e233466..505245cd 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -140,6 +140,12 @@ _installed_or_uninstalled_test_scripts = \ tests/test-signed-commit.sh \ $(NULL) +if USE_LIBSODIUM +_installed_or_uninstalled_test_scripts += \ + tests/test-signed-pull.sh \ + $(NULL) +endif + if USE_GPGME _installed_or_uninstalled_test_scripts += \ tests/test-remote-gpg-import.sh \ diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh new file mode 100755 index 00000000..2f4d4527 --- /dev/null +++ b/tests/test-signed-pull.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Copyright (C) 2019 Collabora Ltd. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..4" + +setup_fake_remote_repo1 "archive" + +repo_mode="archive" + +function repo_init() { + cd ${test_tmpdir} + rm repo -rf + mkdir repo + ostree_repo_init repo --mode=${repo_mode} + ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@" +} + +function test_signed_pull() { + local sign_type="$1" + cd ${test_tmpdir} + ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} \ + -b main -s "A signed commit" --tree=ref=main + + ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u + # make sure gpg verification is correctly on + csum=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo rev-parse main) + objpath=objects/${csum::2}/${csum:2}.commitmeta + remotesig=ostree-srv/gnomerepo/$objpath + localsig=repo/$objpath + mv $remotesig $remotesig.bak + if ${CMD_PREFIX} ostree --repo=repo --depth=0 pull origin main; then + assert_not_reached "pull with sign-verify unexpectedly succeeded?" + fi + # ok now check that we can pull correctly + mv $remotesig.bak $remotesig + ${CMD_PREFIX} ostree --repo=repo pull origin main + echo "ok pull ${sign_type} signed commit" + rm $localsig + ${CMD_PREFIX} ostree --repo=repo pull origin main + test -f $localsig + echo "ok re-pull ${sign_type} signature for stored commit" +} + +DUMMYSIGN="dummysign" +COMMIT_ARGS="--sign=${DUMMYSIGN} --sign-type=dummy" +repo_init --set=sign-verify=true +test_signed_pull "dummy" + + +# Test ostree sign with 'ed25519' module +# Generate private key in PEM format +PEMFILE="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" +openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" + +# Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html +# Extract the private and public parts from generated key. +PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" +SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" +# Secret key is concantination of SEED and PUBLIC +SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" + +echo "SEED = $SEED" +echo "PUBLIC = $PUBLIC" + +COMMIT_ARGS="--sign=${SECRET} --sign-type=ed25519" + +repo_init --set=sign-verify=true +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${PUBLIC}" +test_signed_pull "ed25519" + From 91cc294d05e5b3b942fc11d02cac3eec45a48e40 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 27 Aug 2019 00:28:44 +0300 Subject: [PATCH 027/177] lib/sign: disable mandatory signature check Do not check the signature check by default. Need to enable it explicitly. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index b16b668d..5b9548d7 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3950,7 +3950,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, * explicitly set in the options. */ if (!opt_sign_verify_set) if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, - "sign-verify", TRUE, + "sign-verify", FALSE, &pull_data->sign_verify, error)) goto out; From 073876d9b2e63aa213ad2ae582843682cfb9e9bd Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 27 Aug 2019 00:51:20 +0300 Subject: [PATCH 028/177] lib/sign: add support of file with valid keys for remote Allow to use custom file with public keys for remote. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 42 ++++++++++++++++++++++++++++++-- tests/test-signed-pull.sh | 18 +++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5b9548d7..c3672b78 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1527,6 +1527,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, g_autofree gchar *signature_key = NULL; g_autofree GVariantType *signature_format = NULL; g_autofree gchar *pk_ascii = NULL; + g_autofree gchar *pk_file = NULL; if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) { @@ -1543,7 +1544,25 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, if (!signatures) continue; - /* TODO: load keys for remote here */ + /* Load keys for remote from file */ + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-file", NULL, + &pk_file, NULL); + if (pk_file != NULL) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + g_clear_error (error); + } + + /* Override key if it is set explicitly */ ostree_repo_get_remote_option (pull_data->repo, pull_data->remote_name, "verification-key", NULL, @@ -1931,13 +1950,32 @@ scan_commit_object (OtPullData *pull_data, { g_autoptr (OstreeSign) sign = NULL; g_autofree gchar *pk_ascii = NULL; + g_autofree gchar *pk_file = NULL; if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) { g_clear_error (error); continue; } - /* TODO: load keys for remote here */ + + /* Load keys for remote from file */ + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-file", NULL, + &pk_file, NULL); + if (pk_file != NULL) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + g_clear_error (error); + } + ostree_repo_get_remote_option (pull_data->repo, pull_data->remote_name, "verification-key", NULL, diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 2f4d4527..dc922e81 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..4" +echo "1..7" setup_fake_remote_repo1 "archive" @@ -90,3 +90,19 @@ repo_init --set=sign-verify=true ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${PUBLIC}" test_signed_pull "ed25519" +# Prepare files with public ed25519 signatures +PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" + +# Test the file with multiple keys without a valid public key +for((i=0;i<100;i++)); do + # Generate a list with some public signatures + openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 +done > ${PUBKEYS} +# Add correct key into the list +echo ${PUBLIC} >> ${PUBKEYS} + +repo_init --set=sign-verify=true +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "${PUBKEYS}" +test_signed_pull "ed25519" + +echo "ok verify ed25519 keys file" From 94447617db1b3668a3dce77147008415e7ccde58 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 28 Aug 2019 03:16:22 +0300 Subject: [PATCH 029/177] lib/sign: read ed25519 public keys from well known places If not provided key of file name with keys for remote, then try to use system defaults: - /etc/ostree/trusted.ed25519 - /etc/ostree/trusted.ed25519.d/* - /usr/share/ostree/trusted.ed25519 - /usr/share/ostree/trusted.ed25519.d/* Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 101 +++++++++++++++++++++------- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 299c5441..211e5572 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -479,39 +479,20 @@ err: return FALSE; } -gboolean -ostree_sign_ed25519_load_pk (OstreeSign *self, - GVariant *options, - GError **error) +static gboolean +_load_pk_from_file (OstreeSign *self, + const gchar *filename, + GError **error) { g_debug ("%s enter", __FUNCTION__); - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - g_autoptr (GFile) keyfile = NULL; g_autoptr (GFileInputStream) key_stream_in = NULL; g_autoptr (GDataInputStream) key_data_in = NULL; - const gchar *filename = NULL; - - /* Clear already loaded keys */ - if (sign->public_keys != NULL) - { - g_list_free_full (sign->public_keys, g_object_unref); - sign->public_keys = NULL; - } - - /* Read filename or use will-known if not provided */ - if (! g_variant_lookup (options, "filename", "&s", &filename)) - { - /* TODO: define well-known places to read */ - /* TODO: scan directories */ - filename = "/etc/ostree/trusted.ed25519"; - } - if (!g_file_test (filename, G_FILE_TEST_IS_REGULAR)) { - g_debug ("Can't open file '%s' with pulic keys", filename); + g_debug ("Can't open file '%s' with public keys", filename); g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "File object '%s' is not a regular file", filename); goto err; @@ -533,3 +514,75 @@ err: return FALSE; } +gboolean +ostree_sign_ed25519_load_pk (OstreeSign *self, + GVariant *options, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + gboolean ret = FALSE; + + /* Default paths there to find files with public keys */ + const gchar *default_dirs[] = + { + "/etc/ostree/trusted.ed25519.d", + DATADIR "/ostree/trusted.ed25519.d" + }; + const gchar *default_files[] = + { + "/etc/ostree/trusted.ed25519", + DATADIR "/ostree/trusted.ed25519" + }; + + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + const gchar *filename = NULL; + + /* Clear already loaded keys */ + if (sign->public_keys != NULL) + { + g_list_free_full (sign->public_keys, g_object_unref); + sign->public_keys = NULL; + } + + /* Read only file provided */ + if (g_variant_lookup (options, "filename", "&s", &filename)) + return _load_pk_from_file (self, filename, error); + + /* Scan all well-known files and directories */ + for (gint i=0; i < G_N_ELEMENTS(default_files); i++) + if (!_load_pk_from_file (self, default_files[i], error)) + { + g_debug ("Problem with loading ed25519 public keys from `%s`", default_files[i]); + g_clear_error(error); + } + else + ret = TRUE; + + /* Scan all well-known files and directories */ + for (gint i=0; i < G_N_ELEMENTS(default_dirs); i++) + { + g_autoptr (GDir) dir = g_dir_open (default_dirs[i], 0, error); + if (dir == NULL) + { + g_clear_error (error); + continue; + } + const gchar *entry = NULL; + while ((entry = g_dir_read_name (dir)) != NULL) + { + filename = g_build_filename (default_dirs[i], entry, NULL); + if (!_load_pk_from_file (self, filename, error)) + { + g_debug ("Problem with loading ed25519 public keys from `%s`", filename); + g_clear_error(error); + } + else + ret = TRUE; + } + } + + return ret; +} + From 36e4667973c4b17576758a9ef31d73932bb1d120 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 28 Aug 2019 04:21:22 +0300 Subject: [PATCH 030/177] builtin/sign: allow to sign with keys from secret file Read keys from secret file provided by `--keys-file=` option. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-sign.c | 111 +++++++++++++++++++++++++++-------- tests/test-signed-commit.sh | 17 +++++- 2 files changed, 104 insertions(+), 24 deletions(-) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index 6baeb850..6ce32091 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -48,7 +48,7 @@ static GOptionEntry options[] = { { "verify", 0, 0, G_OPTION_ARG_NONE, &opt_verify, "Verify signatures", NULL}, { "sign-type", 's', 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, #if defined(HAVE_LIBSODIUM) - { "keys-file", 's', 0, G_OPTION_ARG_STRING, &opt_filename, "Read public key(s) from file", "NAME"}, + { "keys-file", 's', 0, G_OPTION_ARG_STRING, &opt_filename, "Read key(s) from file", "NAME"}, #endif { NULL } }; @@ -92,7 +92,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, commit = argv[1]; - if (!opt_verify && argc < 3) + if (!opt_filename && argc < 3) { usage_error (context, "Need at least one KEY-ID to sign with", error); goto out; @@ -174,30 +174,95 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } } - /* Read public signatures from file */ - if (opt_verify && opt_filename) + /* Read signatures from file */ + if (opt_filename) { - g_autoptr (GVariantBuilder) builder = NULL; - g_autoptr (GVariant) options = NULL; - - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); - options = g_variant_builder_end (builder); - - if (!ostree_sign_load_pk (sign, options, error)) + if (opt_verify) { - ret = FALSE; - goto out; + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); + options = g_variant_builder_end (builder); + + if (!ostree_sign_load_pk (sign, options, error)) + { + ret = FALSE; + goto out; + } + if (ostree_sign_commit_verify (sign, + repo, + resolved_commit, + cancellable, + error)) + ret = TRUE; + if (ret != TRUE) + goto out; + } /* Check via file */ + else + { /* Sign with keys from provided file */ + g_autoptr (GFile) keyfile = NULL; + g_autoptr (GFileInputStream) key_stream_in = NULL; + g_autoptr (GDataInputStream) key_data_in = NULL; + + if (!g_file_test (opt_filename, G_FILE_TEST_IS_REGULAR)) + { + g_warning ("Can't open file '%s' with keys", opt_filename); + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "File object '%s' is not a regular file", opt_filename); + goto out; + } + + keyfile = g_file_new_for_path (opt_filename); + key_stream_in = g_file_read (keyfile, NULL, error); + if (key_stream_in == NULL) + goto out; + + key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); + g_assert (key_data_in != NULL); + + /* Use simple file format with just a list of base64 public keys per line */ + while (TRUE) + { + gsize len = 0; + g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); + g_autoptr (GVariant) sk = NULL; + + if (*error != NULL) + goto out; + + if (line == NULL) + goto out; + + + if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) + { + // Just use the string as signature + sk = g_variant_new_string(line); + } + + + if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) + { + gsize key_len = 0; + g_autofree guchar *key = g_base64_decode (line, &key_len); + sk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + } + + if (!ostree_sign_set_sk (sign, sk, error)) + continue; + + ret = ostree_sign_commit (sign, + repo, + resolved_commit, + cancellable, + error); + if (ret != TRUE) + goto out; + } } - if (ostree_sign_commit_verify (sign, - repo, - resolved_commit, - cancellable, - error)) - ret = TRUE; - if (ret != TRUE) - goto out; - } /* Check via file */ + } // No valid signature found if (opt_verify && (ret != TRUE)) diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 2c547542..ce29b1e4 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..7" +echo "1..8" mkdir ${test_tmpdir}/repo ostree_repo_init repo --mode="archive" @@ -57,6 +57,7 @@ if ! has_libsodium; then echo "ok ed25519 signature verified # SKIP due libsodium unavailability" echo "ok multiple signing # SKIP due libsodium unavailability" echo "ok verify ed25519 keys file # SKIP due libsodium unavailability" + echo "ok sign with ed25519 keys file # SKIP due libsodium unavailability" exit 0 fi @@ -136,3 +137,17 @@ echo ${PUBLIC} >> ${PUBKEYS} ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} echo "ok verify ed25519 keys file" + +# Check ed25519 signing with secret file +echo "Unsigned commit for secret file usage" >> file.txt +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo commit -b main -s 'Unsigned commit' +COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" + +KEYFILE="$(mktemp -p ${test_tmpdir} secret_XXXXXX.ed25519)" +echo "${SECRET}" > ${KEYFILE} +# Sign +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 --keys-file=${KEYFILE} ${COMMIT} +# Verify +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} +echo "ok sign with ed25519 keys file" + From 5fc2ddff30c5c8b40e60fd4dd7a6b8c1fab2e98e Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 30 Aug 2019 00:54:17 +0300 Subject: [PATCH 031/177] tests/gpg: skip test in JS if GPG is not supported Skip the single JS test which throws an error if GPG support is disabled in a build time. Signed-off-by: Denis Pynkin --- tests/test-remotes-config-dir.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/test-remotes-config-dir.js b/tests/test-remotes-config-dir.js index 5588116b..f73a82ef 100755 --- a/tests/test-remotes-config-dir.js +++ b/tests/test-remotes-config-dir.js @@ -94,16 +94,22 @@ print("ok add-in-remotes-config-dir"); // Trying to set a remote config option via write_config() for a remote // defined in the config file should succeed -let [, gpg_verify] = repo.remote_get_gpg_verify('bar'); -assertEquals(gpg_verify, true); -repoConfig = repo.copy_config(); -repoConfig.set_boolean('remote "bar"', 'gpg-verify', false); -repo.write_config(repoConfig); -repo.reload_config(null); -[, gpg_verify] = repo.remote_get_gpg_verify('bar'); -assertEquals(gpg_verify, false); - -print("ok config-remote-in-config-file-succeeds"); +try { + let [, gpg_verify] = repo.remote_get_gpg_verify('bar'); + assertEquals(gpg_verify, true); + repoConfig = repo.copy_config(); + repoConfig.set_boolean('remote "bar"', 'gpg-verify', false); + repo.write_config(repoConfig); + repo.reload_config(null); + [, gpg_verify] = repo.remote_get_gpg_verify('bar'); + assertEquals(gpg_verify, false); + print("ok config-remote-in-config-file-succeeds"); +} catch (e) { + // Skip this test if GPG is not supported + if (!(e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_SUPPORTED))) + throw e; + print("ok config-remote-in-config-file-succeeds # SKIP due build without GPG support"); +} // Trying to set a remote config option via write_config() for a remote // defined in the config dir should fail with G_IO_ERROR_EXISTS From 557f42360945b4f70196d7ec4c50443e10918573 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 5 Sep 2019 02:04:25 +0300 Subject: [PATCH 032/177] sign: fix memory leaks and code cleanup Return `const char *` instead of copy of the string -- this allow to avoid unneeded copying and memory leaks in some constructions. Minor code cleanup and optimisations. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 52 ++++++++++++----------------- src/libostree/ostree-sign-dummy.c | 16 ++++----- src/libostree/ostree-sign-dummy.h | 6 ++-- src/libostree/ostree-sign-ed25519.c | 46 +++++++++++-------------- src/libostree/ostree-sign-ed25519.h | 6 ++-- src/libostree/ostree-sign.c | 14 ++++---- src/libostree/ostree-sign.h | 10 +++--- 7 files changed, 65 insertions(+), 85 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c3672b78..b87f6c90 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1519,28 +1519,26 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, gboolean ret = FALSE; g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); /* list all signature types in detached metadata and check if signed by any? */ - g_auto(GStrv) names = ostree_sign_list_names(); + g_auto (GStrv) names = ostree_sign_list_names(); for (guint i=0; i < g_strv_length (names); i++) { g_autoptr (OstreeSign) sign = NULL; + g_autoptr (GError) local_error = NULL; g_autoptr (GVariant) signatures = NULL; - g_autofree gchar *signature_key = NULL; - g_autofree GVariantType *signature_format = NULL; + const gchar *signature_key = NULL; + GVariantType *signature_format = NULL; g_autofree gchar *pk_ascii = NULL; g_autofree gchar *pk_file = NULL; - if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) - { - g_clear_error (error); - continue; - } + if ((sign = ostree_sign_get_by_name (names[i], &local_error)) == NULL) + continue; + signature_key = ostree_sign_metadata_key (sign); signature_format = (GVariantType *) ostree_sign_metadata_format (sign); signatures = g_variant_lookup_value (detached_metadata, signature_key, signature_format); - if (!signatures) continue; @@ -1558,8 +1556,8 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); options = g_variant_builder_end (builder); - if (!ostree_sign_load_pk (sign, options, error)) - g_clear_error (error); + if (!ostree_sign_load_pk (sign, options, &local_error)) + g_clear_error (&local_error); } /* Override key if it is set explicitly */ @@ -1583,27 +1581,23 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } - if (!ostree_sign_set_pk (sign, pk, error)) - g_clear_error (error); + if (!ostree_sign_set_pk (sign, pk, &local_error)) + continue; } /* Set return to true if any sign fit */ if (ostree_sign_metadata_verify (sign, signed_data, signatures, - error + &local_error )) ret = TRUE; - else - g_clear_error (error); } /* Mark the commit as verified to avoid double verification * see process_verify_result () for rationale */ if (ret) - { g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); - } else g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't verify commit"); @@ -1946,17 +1940,15 @@ scan_commit_object (OtPullData *pull_data, gboolean ret = FALSE; /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); - for (guint i=0; i < g_strv_length (names); i++) + for (char **iter=names; iter && *iter; iter++) { g_autoptr (OstreeSign) sign = NULL; + g_autoptr (GError) local_error = NULL; g_autofree gchar *pk_ascii = NULL; g_autofree gchar *pk_file = NULL; - if ((sign = ostree_sign_get_by_name (names[i], error)) == NULL) - { - g_clear_error (error); - continue; - } + if ((sign = ostree_sign_get_by_name (*iter, &local_error)) == NULL) + continue; /* Load keys for remote from file */ ostree_repo_get_remote_option (pull_data->repo, @@ -1972,8 +1964,8 @@ scan_commit_object (OtPullData *pull_data, g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); options = g_variant_builder_end (builder); - if (!ostree_sign_load_pk (sign, options, error)) - g_clear_error (error); + if (!ostree_sign_load_pk (sign, options, &local_error)) + g_clear_error (&local_error); } ostree_repo_get_remote_option (pull_data->repo, @@ -1996,8 +1988,8 @@ scan_commit_object (OtPullData *pull_data, pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } - if (!ostree_sign_set_pk (sign, pk, error)) - g_clear_error (error); + if (!ostree_sign_set_pk (sign, pk, &local_error)) + continue; } @@ -2006,10 +1998,8 @@ scan_commit_object (OtPullData *pull_data, pull_data->repo, checksum, cancellable, - error)) + &local_error)) ret = TRUE; - else - g_clear_error (error); } if (!ret) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 34660cec..fb5a4f9e 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -108,30 +108,26 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, return TRUE; } -gchar * ostree_sign_dummy_get_name (OstreeSign *self) +const gchar * ostree_sign_dummy_get_name (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_autofree gchar *name = g_strdup(OSTREE_SIGN_DUMMY_NAME); - - return g_steal_pointer (&name); + return OSTREE_SIGN_DUMMY_NAME; } -gchar * ostree_sign_dummy_metadata_key (OstreeSign *self) +const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); - g_autofree gchar *key = g_strdup(OSTREE_SIGN_METADATA_DUMMY_KEY); - return g_steal_pointer (&key); + return OSTREE_SIGN_METADATA_DUMMY_KEY; } -gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) +const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); - g_autofree gchar *type = g_strdup(OSTREE_SIGN_METADATA_DUMMY_TYPE); - return g_steal_pointer (&type); + return OSTREE_SIGN_METADATA_DUMMY_TYPE; } gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index 73bad135..847a7313 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -39,7 +39,7 @@ G_DECLARE_FINAL_TYPE (OstreeSignDummy, SIGN_DUMMY, GObject) -gchar * ostree_sign_dummy_get_name (OstreeSign *self); +const gchar * ostree_sign_dummy_get_name (OstreeSign *self); gboolean ostree_sign_dummy_data (OstreeSign *self, GBytes *data, @@ -47,8 +47,8 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, GCancellable *cancellable, GError **error); -gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); -gchar * ostree_sign_dummy_metadata_format (OstreeSign *self); +const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); +const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self); gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, GBytes *data, diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 211e5572..c6c16302 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -112,7 +112,7 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); #ifdef HAVE_LIBSODIUM - g_autofree guchar *sig = NULL; + guchar *sig = NULL; #endif if ((sign->initialized != TRUE) || (sign->secret_key == NULL)) @@ -137,37 +137,33 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, goto err; } - *signature = g_bytes_new (sig, sig_size); + *signature = g_bytes_new_take (sig, sig_size); return TRUE; #endif /* HAVE_LIBSODIUM */ err: return FALSE; } -gchar * ostree_sign_ed25519_get_name (OstreeSign *self) +const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_autofree gchar *name = g_strdup (OSTREE_SIGN_ED25519_NAME); - - return g_steal_pointer (&name); + return OSTREE_SIGN_ED25519_NAME; } -gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) +const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); - g_autofree gchar *key = g_strdup(OSTREE_SIGN_METADATA_ED25519_KEY); - return g_steal_pointer (&key); + return OSTREE_SIGN_METADATA_ED25519_KEY; } -gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) +const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); - g_autofree gchar *type = g_strdup (OSTREE_SIGN_METADATA_ED25519_TYPE); - return g_steal_pointer (&type); + return OSTREE_SIGN_METADATA_ED25519_TYPE; } gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, @@ -187,7 +183,7 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "signature: ed25519: commit have no signatures of my type"); - goto err; + goto out; } if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_ED25519_TYPE)) @@ -195,14 +191,14 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "signature: ed25519: wrong type passed for verification"); - goto err; + goto out; } if (sign->initialized != TRUE) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Not able to verify: libsodium library isn't initialized properly"); - goto err; + goto out; } #ifdef HAVE_LIBSODIUM @@ -217,7 +213,7 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, options = g_variant_builder_end (builder); if (!ostree_sign_ed25519_load_pk (self, options, error)) - goto err; + goto out; } g_debug ("verify: data hash = 0x%x", g_bytes_hash(data)); @@ -259,9 +255,8 @@ gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, "Not able to verify: no valid signatures found"); #endif /* HAVE_LIBSODIUM */ +out: return ret; -err: - return FALSE; } gboolean @@ -312,7 +307,6 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - g_autofree char * hex = NULL; g_free (sign->secret_key); @@ -326,9 +320,6 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, goto err; } - hex = g_malloc0 (crypto_sign_SECRETKEYBYTES*2 + 1); -// g_debug ("Set ed25519 secret key = %s", sodium_bin2hex (hex, crypto_sign_SECRETKEYBYTES*2+1, sign->secret_key, n_elements)); - return TRUE; err: @@ -348,7 +339,7 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, /* Substitute the key(s) with a new one */ if (sign->public_keys != NULL) { - g_list_free_full (sign->public_keys, g_object_unref); + g_list_free_full (sign->public_keys, g_free); sign->public_keys = NULL; } @@ -380,9 +371,11 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, goto err; } - key = g_memdup (key, n_elements); if (g_list_find (sign->public_keys, key) == NULL) - sign->public_keys = g_list_prepend (sign->public_keys, key); + { + gpointer newkey = g_memdup (key, n_elements); + sign->public_keys = g_list_prepend (sign->public_keys, newkey); + } return TRUE; @@ -485,6 +478,7 @@ _load_pk_from_file (OstreeSign *self, GError **error) { g_debug ("%s enter", __FUNCTION__); + g_debug ("Processing file '%s'", filename); g_autoptr (GFile) keyfile = NULL; g_autoptr (GFileInputStream) key_stream_in = NULL; @@ -542,7 +536,7 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, /* Clear already loaded keys */ if (sign->public_keys != NULL) { - g_list_free_full (sign->public_keys, g_object_unref); + g_list_free_full (sign->public_keys, g_free); sign->public_keys = NULL; } diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index 797ac138..eb8f6701 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -46,9 +46,9 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, GCancellable *cancellable, GError **error); -gchar * ostree_sign_ed25519_get_name (OstreeSign *self); -gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); -gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); +const gchar * ostree_sign_ed25519_get_name (OstreeSign *self); +const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); +const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, GBytes *data, diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 5f7078f1..6e67acaa 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -71,7 +71,7 @@ ostree_sign_default_init (OstreeSignInterface *iface) g_debug ("OstreeSign initialization"); } -gchar * ostree_sign_metadata_key (OstreeSign *self) +const gchar * ostree_sign_metadata_key (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); @@ -79,7 +79,7 @@ gchar * ostree_sign_metadata_key (OstreeSign *self) return OSTREE_SIGN_GET_IFACE (self)->metadata_key (self); } -gchar * ostree_sign_metadata_format (OstreeSign *self) +const gchar * ostree_sign_metadata_format (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); @@ -134,7 +134,7 @@ ostree_sign_load_pk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->load_pk == NULL) - return FALSE; + return TRUE; return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, options, error); } @@ -170,8 +170,8 @@ ostree_sign_detached_metadata_append (OstreeSign *self, g_variant_dict_init (&metadata_dict, existing_metadata); - g_autofree gchar *signature_key = ostree_sign_metadata_key(self); - g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); + const gchar *signature_key = ostree_sign_metadata_key(self); + GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); signature_data = g_variant_dict_lookup_value (&metadata_dict, signature_key, @@ -234,8 +234,8 @@ ostree_sign_commit_verify (OstreeSign *self, g_autoptr(GVariant) signatures = NULL; - g_autofree gchar *signature_key = ostree_sign_metadata_key(self); - g_autofree GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); + const gchar *signature_key = ostree_sign_metadata_key(self); + GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format(self); if (metadata) signatures = g_variant_lookup_value (metadata, diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 78e2487d..a9648cb1 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -47,14 +47,14 @@ G_DECLARE_INTERFACE (OstreeSign, ostree_sign, OSTREE, SIGN, GObject) struct _OstreeSignInterface { GTypeInterface g_iface; - gchar *(* get_name) (OstreeSign *self); + const gchar *(* get_name) (OstreeSign *self); gboolean (* data) (OstreeSign *self, GBytes *data, GBytes **signature, GCancellable *cancellable, GError **error); - gchar *(* metadata_key) (OstreeSign *self); - gchar *(* metadata_format) (OstreeSign *self); + const gchar *(* metadata_key) (OstreeSign *self); + const gchar *(* metadata_format) (OstreeSign *self); gboolean (* metadata_verify) (OstreeSign *self, GBytes *data, GVariant *metadata, @@ -90,10 +90,10 @@ gboolean ostree_sign_data (OstreeSign *self, _OSTREE_PUBLIC -gchar * ostree_sign_metadata_key (OstreeSign *self); +const gchar * ostree_sign_metadata_key (OstreeSign *self); _OSTREE_PUBLIC -gchar * ostree_sign_metadata_format (OstreeSign *self); +const gchar * ostree_sign_metadata_format (OstreeSign *self); _OSTREE_PUBLIC GVariant * ostree_sign_detached_metadata_append (OstreeSign *self, From ea291a0605999242491fed982430c25aebbb2f9e Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 5 Sep 2019 16:33:52 +0300 Subject: [PATCH 033/177] builtin/sign: allow to use multiple public keys for verification `ostree sign` is able to use several public keys provided via arguments and via file with keys. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-sign.c | 62 ++++++++++++++++++++---------------- tests/test-signed-commit.sh | 18 +++++++++++ 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index 6ce32091..b1c9a73b 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -64,8 +64,8 @@ usage_error (GOptionContext *context, const char *message, GError **error) gboolean ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { - g_autoptr(GOptionContext) context = NULL; - g_autoptr(OstreeRepo) repo = NULL; + g_autoptr (GOptionContext) context = NULL; + g_autoptr (OstreeRepo) repo = NULL; g_autoptr (OstreeSign) sign = NULL; g_autofree char *resolved_commit = NULL; const char *commit; @@ -92,7 +92,10 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, commit = argv[1]; - if (!opt_filename && argc < 3) + /* Verification could be done via system files with public keys */ + if (!opt_verify && + !opt_filename && + argc < 3) { usage_error (context, "Need at least one KEY-ID to sign with", error); goto out; @@ -110,10 +113,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, sign = ostree_sign_get_by_name (opt_sign_name, error); if (sign == NULL) - { - ret = FALSE; - goto out; - } + goto out; for (ii = 0; ii < n_key_ids; ii++) { @@ -129,6 +129,9 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } if (opt_verify) { + g_autoptr (GError) local_error = NULL; + + if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) { gsize key_len = 0; @@ -136,17 +139,14 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); } - if (!ostree_sign_set_pk (sign, pk, error)) - { - ret = FALSE; - goto out; - } + if (!ostree_sign_set_pk (sign, pk, &local_error)) + continue; if (ostree_sign_commit_verify (sign, repo, resolved_commit, cancellable, - error)) + &local_error)) ret = TRUE; } else @@ -174,34 +174,36 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } } - /* Read signatures from file */ - if (opt_filename) + /* Try to verify with user-provided file or system configuration */ + if (opt_verify) { - if (opt_verify) + if ((n_key_ids == 0) || opt_filename) { g_autoptr (GVariantBuilder) builder = NULL; g_autoptr (GVariant) options = NULL; builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); + /* The last chance for verification source -- system files */ + if (opt_filename) + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); options = g_variant_builder_end (builder); if (!ostree_sign_load_pk (sign, options, error)) - { - ret = FALSE; - goto out; - } + goto out; + if (ostree_sign_commit_verify (sign, repo, resolved_commit, cancellable, error)) ret = TRUE; - if (ret != TRUE) - goto out; } /* Check via file */ - else - { /* Sign with keys from provided file */ + } + else + { + /* Sign with keys from provided file */ + if (opt_filename) + { g_autoptr (GFile) keyfile = NULL; g_autoptr (GFileInputStream) key_stream_in = NULL; g_autoptr (GDataInputStream) key_data_in = NULL; @@ -233,7 +235,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, goto out; if (line == NULL) - goto out; + break; if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) @@ -251,7 +253,10 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } if (!ostree_sign_set_sk (sign, sk, error)) - continue; + { + ret = FALSE; + goto out; + } ret = ostree_sign_commit (sign, repo, @@ -271,5 +276,8 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, "No valid signatures found"); out: + /* It is possible to have an error due multiple signatures check */ + if (ret == TRUE) + g_clear_error (error); return ret; } diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index ce29b1e4..e4138817 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -73,6 +73,8 @@ SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" # Secret key is concantination of SEED and PUBLIC SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" +WRONG_PUBLIC="$(openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64)" + echo "SEED = $SEED" echo "PUBLIC = $PUBLIC" @@ -85,7 +87,15 @@ ${CMD_PREFIX} ostree --repo=repo show ${COMMIT} --print-detached-metadata-key=os echo "ok Detached ed25519 signature added" # Verify vith sign mechanism +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC}; then + exit 1 +fi ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${WRONG_PUBLIC} ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} ${WRONG_PUBLIC} ${WRONG_PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${WRONG_PUBLIC} ${PUBLIC} ${WRONG_PUBLIC} ${WRONG_PUBLIC} echo "ok ed25519 signature verified" # Check if we able to use all available modules to sign the same commit @@ -128,9 +138,17 @@ for((i=0;i<100;i++)); do # Generate a list with some public signatures openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 done > ${PUBKEYS} +# Check if file contain no valid signatures if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then exit 1 fi +# Check if no valid signatures provided via args&file +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} ${WRONG_PUBLIC}; then + exit 1 +fi + +#Test keys file and public key +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} ${PUBLIC} # Add correct key into the list echo ${PUBLIC} >> ${PUBKEYS} From 95ab57c17ee2fcc93ee53b9a46ee6ed0a1c07b2b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 6 Oct 2019 23:40:04 +0300 Subject: [PATCH 034/177] lib/sign-ed25519: cleanup unneeded code Removed unused code. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 1 - src/libostree/libostree-devel.sym | 1 - src/libostree/ostree-sign-ed25519.c | 89 ----------------------------- src/libostree/ostree-sign-ed25519.h | 21 +++---- 4 files changed, 7 insertions(+), 105 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 806bd1a7..1ea6e548 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -723,7 +723,6 @@ ostree_sign_add_pk ostree_sign_load_pk ostree_sign_set_pk ostree_sign_set_sk -ostree_sign_ed25519_keypair_generate ostree_sign_get_type ostree_sign_dummy_get_type diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 8be5a3bf..93f904b1 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -37,7 +37,6 @@ global: ostree_sign_set_sk; ostree_sign_dummy_get_type; ostree_sign_ed25519_get_type; - ostree_sign_ed25519_keypair_generate; } LIBOSTREE_2020.1; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index c6c16302..1fb6ae05 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -38,12 +38,6 @@ #define OSTREE_SIGN_METADATA_ED25519_KEY "ostree.sign.ed25519" #define OSTREE_SIGN_METADATA_ED25519_TYPE "aay" -#if 0 -#define SIGNIFY_COMMENT_HEADER "untrusted comment:" -#define SIGNIFY_ID_LENGTH 8 -#define SIGNIFY_MAGIC_ED25519 "Ed" -#endif - struct _OstreeSignEd25519 { GObject parent; @@ -259,45 +253,6 @@ out: return ret; } -gboolean -ostree_sign_ed25519_keypair_generate (OstreeSign *self, - GVariant **out_secret_key, - GVariant **out_public_key, - GError **error) - { - g_debug ("%s enter", __FUNCTION__); - g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - - if (sign->initialized != TRUE) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to sign -- libsodium library isn't initialized properly"); - goto err; - } - -#ifdef HAVE_LIBSODIUM - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - - if (crypto_sign_keypair(pk, sk)) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to generate keypair"); - goto err; - } - - *out_secret_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, sk, crypto_sign_SECRETKEYBYTES, sizeof(guchar)); - *out_public_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, pk, crypto_sign_PUBLICKEYBYTES, sizeof(guchar)); - - return TRUE; -#endif /* HAVE_LIBSODIUM */ - -err: - return FALSE; -} - gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) @@ -392,50 +347,6 @@ _load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError ** #ifdef HAVE_LIBSODIUM gboolean ret = FALSE; -#if 0 -/* Try to load the public key in signify format from the stream - * https://www.openbsd.org/papers/bsdcan-signify.html - * - * FIXME: Not sure if we need to support that format. - * */ - g_autofree gchar * comment = NULL; - while (TRUE) - { - gsize len = 0; - g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); - if (error) - goto err; - - if (line) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Signify format for ed25519 public key not found"); - goto err; - } - - if (comment == NULL) - { - /* Scan for the comment first and compare with prefix&suffix */ - if (g_str_has_prefix (line, SIGNIFY_COMMENT_HEADER) && g_str_has_suffix (line, "public key")) - /* Save comment without the prefix and blank space */ - comment = g_strdup (line + strlen(SIGNIFY_COMMENT_HEADER) + 1); - } - else - { - /* Read the key itself */ - /* base64 encoded key */ - gsize keylen = 0; - g_autofree guchar *key = g_base64_decode (line, &keylen); - - /* Malformed key */ - if (keylen != SIGNIFY_ID_LENGTH || - strncmp (line, SIGNIFY_MAGIC_ED25519, strlen(SIGNIFY_MAGIC_ED25519)) != 0) - continue; - - } - } -#endif /* 0 */ - /* Use simple file format with just a list of base64 public keys per line */ while (TRUE) { diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index eb8f6701..16da4828 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -41,19 +41,19 @@ G_DECLARE_FINAL_TYPE (OstreeSignEd25519, gboolean ostree_sign_ed25519_data (OstreeSign *self, - GBytes *data, - GBytes **signature, - GCancellable *cancellable, - GError **error); + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); const gchar * ostree_sign_ed25519_get_name (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error); + GBytes *data, + GVariant *signatures, + GError **error); gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, @@ -71,12 +71,5 @@ gboolean ostree_sign_ed25519_load_pk (OstreeSign *self, GVariant *options, GError **error); -_OSTREE_PUBLIC -gboolean ostree_sign_ed25519_keypair_generate (OstreeSign *self, - GVariant **out_secret_key, - GVariant **out_public_key, - GError **error); - - G_END_DECLS From bc4488692c68a2148b921efe94ea4dc2cee24725 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 7 Oct 2019 02:59:15 +0300 Subject: [PATCH 035/177] lib/sign: public API optimisation Removed from public `ostree_sign_detached_metadata_append` function. Renamed `metadata_verify` into `data_verify` to fit to real functionality. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 3 +- src/libostree/libostree-devel.sym | 3 +- src/libostree/ostree-repo-pull.c | 2 +- src/libostree/ostree-sign-dummy.c | 12 +++---- src/libostree/ostree-sign-dummy.h | 12 +++---- src/libostree/ostree-sign-ed25519.c | 54 ++++++++++++++--------------- src/libostree/ostree-sign-ed25519.h | 10 +++--- src/libostree/ostree-sign.c | 41 ++++++++++------------ src/libostree/ostree-sign.h | 29 +++++----------- 9 files changed, 75 insertions(+), 91 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 1ea6e548..440338c2 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -715,10 +715,9 @@ ostree_sign_list_names ostree_sign_commit ostree_sign_commit_verify ostree_sign_data +ostree_sign_data_verify ostree_sign_get_by_name ostree_sign_get_name -ostree_sign_detached_metadata_append -ostree_sign_metadata_verify ostree_sign_add_pk ostree_sign_load_pk ostree_sign_set_pk diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 93f904b1..3ca8f2c6 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -27,10 +27,9 @@ global: ostree_sign_commit; ostree_sign_commit_verify; ostree_sign_data; + ostree_sign_data_verify; ostree_sign_get_by_name; ostree_sign_get_name; - ostree_sign_detached_metadata_append; - ostree_sign_metadata_verify; ostree_sign_load_pk; ostree_sign_set_pk; ostree_sign_add_pk; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index b87f6c90..1a4e64da 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1586,7 +1586,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, } /* Set return to true if any sign fit */ - if (ostree_sign_metadata_verify (sign, + if (ostree_sign_data_verify (sign, signed_data, signatures, &local_error diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index fb5a4f9e..e2d1fe56 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -54,13 +54,13 @@ ostree_sign_dummy_iface_init (OstreeSignInterface *self) { g_debug ("%s enter", __FUNCTION__); - self->data = ostree_sign_dummy_data; self->get_name = ostree_sign_dummy_get_name; + self->data = ostree_sign_dummy_data; + self->data_verify = ostree_sign_dummy_data_verify; self->metadata_key = ostree_sign_dummy_metadata_key; self->metadata_format = ostree_sign_dummy_metadata_format; - self->metadata_verify = ostree_sign_dummy_metadata_verify; - self->set_sk = ostree_sign_dummy_set_signature; - self->set_pk = ostree_sign_dummy_set_signature; + self->set_sk = ostree_sign_dummy_set_key; + self->set_pk = ostree_sign_dummy_set_key; } static void @@ -77,7 +77,7 @@ ostree_sign_dummy_init (OstreeSignDummy *self) self->signature_ascii = g_strdup(OSTREE_SIGN_DUMMY_SIGNATURE); } -gboolean ostree_sign_dummy_set_signature (OstreeSign *self, GVariant *key, GError **error) +gboolean ostree_sign_dummy_set_key (OstreeSign *self, GVariant *key, GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -130,7 +130,7 @@ const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) return OSTREE_SIGN_METADATA_DUMMY_TYPE; } -gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, +gboolean ostree_sign_dummy_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, GError **error) diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index 847a7313..a0d10e1d 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -47,15 +47,15 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, GCancellable *cancellable, GError **error); +gboolean ostree_sign_dummy_data_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); + const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self); -gboolean ostree_sign_dummy_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error); - -gboolean ostree_sign_dummy_set_signature (OstreeSign *self, GVariant *key, GError **error); +gboolean ostree_sign_dummy_set_key (OstreeSign *self, GVariant *key, GError **error); G_END_DECLS diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 1fb6ae05..2bf10cf1 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -58,10 +58,10 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) g_debug ("%s enter", __FUNCTION__); self->data = ostree_sign_ed25519_data; + self->data_verify = ostree_sign_ed25519_data_verify; self->get_name = ostree_sign_ed25519_get_name; self->metadata_key = ostree_sign_ed25519_metadata_key; self->metadata_format = ostree_sign_ed25519_metadata_format; - self->metadata_verify = ostree_sign_ed25519_metadata_verify; self->set_sk = ostree_sign_ed25519_set_sk; self->set_pk = ostree_sign_ed25519_set_pk; self->add_pk = ostree_sign_ed25519_add_pk; @@ -138,32 +138,10 @@ err: return FALSE; } -const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) -{ - g_debug ("%s enter", __FUNCTION__); - g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - - return OSTREE_SIGN_ED25519_NAME; -} - -const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) -{ - g_debug ("%s enter", __FUNCTION__); - - return OSTREE_SIGN_METADATA_ED25519_KEY; -} - -const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) -{ - g_debug ("%s enter", __FUNCTION__); - - return OSTREE_SIGN_METADATA_ED25519_TYPE; -} - -gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error) +gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -253,6 +231,28 @@ out: return ret; } +const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + return OSTREE_SIGN_ED25519_NAME; +} + +const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + return OSTREE_SIGN_METADATA_ED25519_KEY; +} + +const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) +{ + g_debug ("%s enter", __FUNCTION__); + + return OSTREE_SIGN_METADATA_ED25519_TYPE; +} + gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index 16da4828..4519961d 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -46,15 +46,15 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, GCancellable *cancellable, GError **error); +gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); + const gchar * ostree_sign_ed25519_get_name (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); -gboolean ostree_sign_ed25519_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error); - gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error); diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 6e67acaa..b1975215 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -153,13 +153,26 @@ gboolean ostree_sign_data (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->data (self, data, signature, cancellable, error); } +gboolean +ostree_sign_data_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data_verify != NULL, FALSE); + + return OSTREE_SIGN_GET_IFACE (self)->data_verify(self, data, signatures, error); +} + /* * Adopted version of _ostree_detached_metadata_append_gpg_sig () */ -GVariant * -ostree_sign_detached_metadata_append (OstreeSign *self, - GVariant *existing_metadata, - GBytes *signature_bytes) +static GVariant * +_sign_detached_metadata_append (OstreeSign *self, + GVariant *existing_metadata, + GBytes *signature_bytes) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (signature_bytes != NULL, FALSE); @@ -189,20 +202,6 @@ ostree_sign_detached_metadata_append (OstreeSign *self, return g_variant_dict_end (&metadata_dict); } - -gboolean -ostree_sign_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error) -{ - g_debug ("%s enter", __FUNCTION__); - g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_verify != NULL, FALSE); - - return OSTREE_SIGN_GET_IFACE (self)->metadata_verify(self, data, signatures, error); -} - gboolean ostree_sign_commit_verify (OstreeSign *self, OstreeRepo *repo, @@ -243,7 +242,7 @@ ostree_sign_commit_verify (OstreeSign *self, signature_format); - return ostree_sign_metadata_verify (self, + return ostree_sign_data_verify (self, signed_data, signatures, error); @@ -325,8 +324,6 @@ ostree_sign_commit (OstreeSign *self, error)) return glnx_prefix_error (error, "Failed to read detached metadata"); - // TODO: d4s: check if already signed? - commit_data = g_variant_get_data_as_bytes (commit_variant); if (!ostree_sign_data (self, commit_data, &signature, @@ -334,7 +331,7 @@ ostree_sign_commit (OstreeSign *self, return glnx_prefix_error (error, "Not able to sign the cobject"); new_metadata = - ostree_sign_detached_metadata_append (self, old_metadata, signature); + _sign_detached_metadata_append (self, old_metadata, signature); if (!ostree_repo_write_commit_detached_metadata (repo, commit_checksum, diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index a9648cb1..008c3f9d 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -53,29 +53,24 @@ struct _OstreeSignInterface GBytes **signature, GCancellable *cancellable, GError **error); + gboolean (* data_verify) (OstreeSign *self, + GBytes *data, + GVariant *metadata, + GError **error); const gchar *(* metadata_key) (OstreeSign *self); const gchar *(* metadata_format) (OstreeSign *self); - gboolean (* metadata_verify) (OstreeSign *self, - GBytes *data, - GVariant *metadata, - GError **error); - gboolean (* set_sk) (OstreeSign *self, GVariant *secret_key, GError **error); - gboolean (* set_pk) (OstreeSign *self, GVariant *public_key, GError **error); - gboolean (* add_pk) (OstreeSign *self, GVariant *public_key, GError **error); - gboolean (* load_pk) (OstreeSign *self, GVariant *options, GError **error); - }; _OSTREE_PUBLIC @@ -88,6 +83,11 @@ gboolean ostree_sign_data (OstreeSign *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sign_data_verify (OstreeSign *self, + GBytes *data, + GVariant *signatures, + GError **error); _OSTREE_PUBLIC const gchar * ostree_sign_metadata_key (OstreeSign *self); @@ -95,11 +95,6 @@ const gchar * ostree_sign_metadata_key (OstreeSign *self); _OSTREE_PUBLIC const gchar * ostree_sign_metadata_format (OstreeSign *self); -_OSTREE_PUBLIC -GVariant * ostree_sign_detached_metadata_append (OstreeSign *self, - GVariant *existing_metadata, - GBytes *signature_bytes); - _OSTREE_PUBLIC gboolean ostree_sign_commit (OstreeSign *self, OstreeRepo *repo, @@ -107,12 +102,6 @@ gboolean ostree_sign_commit (OstreeSign *self, GCancellable *cancellable, GError **error); -_OSTREE_PUBLIC -gboolean ostree_sign_metadata_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error); - _OSTREE_PUBLIC gboolean ostree_sign_commit_verify (OstreeSign *self, OstreeRepo *repo, From f0181adff3d2a494944f86a9ec248d4763498045 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 7 Oct 2019 23:37:08 +0300 Subject: [PATCH 036/177] lib/sign: allow to add keys as base64 string for ed25519 Allow to add public and secret key for ed25519 module as based64 string. This allows to use common API for pulling and builtins without knowledge of used signature algorithm. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 32 +++----------------- src/libostree/ostree-sign-ed25519.c | 47 +++++++++++++++++++++++++++-- src/ostree/ot-builtin-commit.c | 12 +------- src/ostree/ot-builtin-sign.c | 45 ++++----------------------- 4 files changed, 56 insertions(+), 80 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 1a4e64da..f3f13ed3 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1567,20 +1567,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, &pk_ascii, NULL); if (pk_ascii != NULL) { - g_autoptr (GVariant) pk = NULL; - - if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) - { - // Just use the string as signature - pk = g_variant_new_string(pk_ascii); - } - else if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (pk_ascii, &key_len); - pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } - + g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); if (!ostree_sign_set_pk (sign, pk, &local_error)) continue; } @@ -1976,18 +1963,8 @@ scan_commit_object (OtPullData *pull_data, { g_autoptr (GVariant) pk = NULL; - if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) - { - // Just use the string as signature - pk = g_variant_new_string(pk_ascii); - } - else if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (pk_ascii, &key_len); - pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } - + // Just use the string as signature + pk = g_variant_new_string(pk_ascii); if (!ostree_sign_set_pk (sign, pk, &local_error)) continue; } @@ -4853,11 +4830,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, else gpg_verify_state = (pull_data->gpg_verify ? "commit" : "disabled"); - g_string_append_printf (msg, "\nsecurity: GPG: %s ", gpg_verify_state); #else gpg_verify_state = "disabled"; - g_string_append_printf (msg, "\nsecurity: %s ", gpg_verify_state); #endif /* OSTREE_DISABLE_GPGME */ + g_string_append_printf (msg, "\nsecurity: GPG: %s ", gpg_verify_state); const char *sign_verify_state; sign_verify_state = (pull_data->sign_verify ? "commit" : "disabled"); diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 2bf10cf1..f90a310c 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -253,6 +253,10 @@ const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) return OSTREE_SIGN_METADATA_ED25519_TYPE; } +/* Support 2 representations: + * base64 ascii -- secret key is passed as string + * raw key -- key is passed as bytes array + * */ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) @@ -266,7 +270,23 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, g_free (sign->secret_key); gsize n_elements = 0; - sign->secret_key = (guchar *) g_variant_get_fixed_array (secret_key, &n_elements, sizeof(guchar)); + + if (g_variant_is_of_type (secret_key, G_VARIANT_TYPE_STRING)) + { + const gchar *sk_ascii = g_variant_get_string (secret_key, NULL); + sign->secret_key = g_base64_decode (sk_ascii, &n_elements); + } + else if (g_variant_is_of_type (secret_key, G_VARIANT_TYPE_BYTESTRING)) + { + sign->secret_key = (guchar *) g_variant_get_fixed_array (secret_key, &n_elements, sizeof(guchar)); + } + else + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unknown ed25519 secret key type"); + goto err; + } + if (n_elements != crypto_sign_SECRETKEYBYTES) { @@ -282,6 +302,10 @@ err: return FALSE; } +/* Support 2 representations: + * base64 ascii -- public key is passed as string + * raw key -- key is passed as bytes array + * */ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, GVariant *public_key, GError **error) @@ -301,6 +325,10 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, return ostree_sign_ed25519_add_pk (self, public_key, error); } +/* Support 2 representations: + * base64 ascii -- public key is passed as string + * raw key -- key is passed as bytes array + * */ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, GVariant *public_key, GError **error) @@ -314,7 +342,22 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, gpointer key = NULL; gsize n_elements = 0; - key = (gpointer) g_variant_get_fixed_array (public_key, &n_elements, sizeof(guchar)); + + if (g_variant_is_of_type (public_key, G_VARIANT_TYPE_STRING)) + { + const gchar *pk_ascii = g_variant_get_string (public_key, NULL); + key = g_base64_decode (pk_ascii, &n_elements); + } + else if (g_variant_is_of_type (public_key, G_VARIANT_TYPE_BYTESTRING)) + { + key = (gpointer) g_variant_get_fixed_array (public_key, &n_elements, sizeof(guchar)); + } + else + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unknown ed25519 public key type"); + goto err; + } hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 7d412639..606af2be 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -855,17 +855,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio const char *keyid = *iter; g_autoptr (GVariant) secret_key = NULL; - if (!g_strcmp0(ostree_sign_get_name (sign), "dummy")) - { - secret_key = g_variant_new_string (keyid); - } - else if (!g_strcmp0 (ostree_sign_get_name (sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (keyid, &key_len); - - secret_key = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } + secret_key = g_variant_new_string (keyid); if (!ostree_sign_set_sk (sign, secret_key, error)) goto out; diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index b1c9a73b..f673631d 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -72,11 +72,6 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, char **key_ids; int n_key_ids, ii; gboolean ret = FALSE; -#if defined(HAVE_LIBSODIUM) - g_autoptr (GVariant) ed25519_sk = NULL; - g_autoptr (GVariant) ed25519_pk = NULL; -#endif - context = g_option_context_new ("COMMIT KEY-ID..."); @@ -119,25 +114,14 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, { g_autoptr (GVariant) sk = NULL; g_autoptr (GVariant) pk = NULL; - g_autofree guchar *key = NULL; - if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) - { - // Just use the string as signature - sk = g_variant_new_string(key_ids[ii]); - pk = g_variant_new_string(key_ids[ii]); - } if (opt_verify) { g_autoptr (GError) local_error = NULL; - if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); - pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } + // Pass the key as a string + pk = g_variant_new_string(key_ids[ii]); if (!ostree_sign_set_pk (sign, pk, &local_error)) continue; @@ -151,13 +135,8 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } else { - if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (key_ids[ii], &key_len); - sk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } - + // Pass the key as a string + sk = g_variant_new_string(key_ids[ii]); if (!ostree_sign_set_sk (sign, sk, error)) { ret = FALSE; @@ -238,20 +217,8 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, break; - if (!g_strcmp0(ostree_sign_get_name(sign), "dummy")) - { - // Just use the string as signature - sk = g_variant_new_string(line); - } - - - if (!g_strcmp0(ostree_sign_get_name(sign), "ed25519")) - { - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (line, &key_len); - sk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); - } - + // Pass the key as a string + sk = g_variant_new_string(line); if (!ostree_sign_set_sk (sign, sk, error)) { ret = FALSE; From 7fa7c3c4f94ebd0e17fa831e47e53cf4aa3e94e0 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 8 Oct 2019 01:55:25 +0300 Subject: [PATCH 037/177] sign: use common function for loading public keys during pulling Add function `_load_public_keys()` to pre-load public keys according remote's configuration. If no keys configured for remote, then use system-wide configuration. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 184 +++++++++++++++++++------------ 1 file changed, 113 insertions(+), 71 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f3f13ed3..4dd0f11a 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1469,6 +1469,104 @@ process_verify_result (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ +/* _remote_load_public_keys: + * + * Load public keys according remote's configuration: + * inlined key passed via config option `verification-key` or + * file name with public keys via `verification-file` option. + * + * If both options are set then load all all public keys + * both from file and inlined in config. + * + * Returns: %FALSE if any source is configured but nothing has been loaded. + * Returns: %TRUE if no configuration or any key loaded. + * */ +static gboolean +_load_public_keys (OtPullData *pull_data, + OstreeSign *sign) +{ + + g_autofree gchar *pk_ascii = NULL; + g_autofree gchar *pk_file = NULL; + gboolean loaded_from_file = TRUE; + gboolean loaded_inlined = TRUE; + g_autoptr (GError) error = NULL; + + /* Load keys for remote from file */ + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-file", NULL, + &pk_file, NULL); + + ostree_repo_get_remote_option (pull_data->repo, + pull_data->remote_name, + "verification-key", NULL, + &pk_ascii, NULL); + + /* return TRUE if there is no configuration for remote */ + if ((pk_file == NULL) &&(pk_ascii == NULL)) + { + /* It is expected what remote may have verification file as + * a part of configuration. Hence there is not a lot of sense + * for automatic resolve of per-remote keystore file as it + * used in find_keyring () for GPG. + * If it is needed to add the similar mechanism, it is preferable + * to pass the path to ostree_sign_load_pk () via GVariant options + * and call it here for loading with method and file structure + * specific for signature type. + */ + return TRUE; + } + + if (pk_file != NULL) + { + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); + options = g_variant_builder_end (builder); + + if (ostree_sign_load_pk (sign, options, &error)) + loaded_from_file = TRUE; + else + { + if (error == NULL) + g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "unknown reason"); + + g_warning("Unable to load public keys from file '%s': %s", + pk_file, error->message); + g_clear_error (&error); + } + } + + if (pk_ascii != NULL) + { + g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); + + /* Add inlined public key */ + if (loaded_from_file) + loaded_inlined = ostree_sign_add_pk (sign, pk, &error); + else + loaded_inlined = ostree_sign_set_pk (sign, pk, &error); + + if (!loaded_inlined) + { + if (error == NULL) + g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, + "unknown reason"); + + g_warning("Unable to load public key '%s': %s", + pk_ascii, error->message); + g_clear_error (&error); + } + } + + /* Return true if able to load from any source */ + return (loaded_from_file || loaded_inlined); +} + static gboolean ostree_verify_unwritten_commit (OtPullData *pull_data, const char *checksum, @@ -1518,17 +1616,16 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, gboolean ret = FALSE; g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); + /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); for (guint i=0; i < g_strv_length (names); i++) { g_autoptr (OstreeSign) sign = NULL; - g_autoptr (GError) local_error = NULL; g_autoptr (GVariant) signatures = NULL; const gchar *signature_key = NULL; GVariantType *signature_format = NULL; - g_autofree gchar *pk_ascii = NULL; - g_autofree gchar *pk_file = NULL; + g_autoptr (GError) local_error = NULL; if ((sign = ostree_sign_get_by_name (names[i], &local_error)) == NULL) continue; @@ -1539,45 +1636,21 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, signatures = g_variant_lookup_value (detached_metadata, signature_key, signature_format); + + /* If not found signatures for requested signature subsystem */ if (!signatures) continue; - /* Load keys for remote from file */ - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, - "verification-file", NULL, - &pk_file, NULL); - if (pk_file != NULL) - { - g_autoptr (GVariantBuilder) builder = NULL; - g_autoptr (GVariant) options = NULL; + /* Try to load public key(s) according remote's configuration */ + if (!_load_public_keys (pull_data, sign)) + continue; - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); - options = g_variant_builder_end (builder); - - if (!ostree_sign_load_pk (sign, options, &local_error)) - g_clear_error (&local_error); - } - - /* Override key if it is set explicitly */ - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, - "verification-key", NULL, - &pk_ascii, NULL); - if (pk_ascii != NULL) - { - g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); - if (!ostree_sign_set_pk (sign, pk, &local_error)) - continue; - } - - /* Set return to true if any sign fit */ + /* Return true if any signature fit to pre-loaded public keys. + * If no keys configured -- then system configuration will be used */ if (ostree_sign_data_verify (sign, - signed_data, - signatures, - &local_error - )) + signed_data, + signatures, + &local_error)) ret = TRUE; } @@ -1931,44 +2004,13 @@ scan_commit_object (OtPullData *pull_data, { g_autoptr (OstreeSign) sign = NULL; g_autoptr (GError) local_error = NULL; - g_autofree gchar *pk_ascii = NULL; - g_autofree gchar *pk_file = NULL; if ((sign = ostree_sign_get_by_name (*iter, &local_error)) == NULL) continue; - /* Load keys for remote from file */ - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, - "verification-file", NULL, - &pk_file, NULL); - if (pk_file != NULL) - { - g_autoptr (GVariantBuilder) builder = NULL; - g_autoptr (GVariant) options = NULL; - - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); - options = g_variant_builder_end (builder); - - if (!ostree_sign_load_pk (sign, options, &local_error)) - g_clear_error (&local_error); - } - - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, - "verification-key", NULL, - &pk_ascii, NULL); - if (pk_ascii != NULL) - { - g_autoptr (GVariant) pk = NULL; - - // Just use the string as signature - pk = g_variant_new_string(pk_ascii); - if (!ostree_sign_set_pk (sign, pk, &local_error)) - continue; - } - + /* Try to load public key(s) according remote's configuration */ + if (!_load_public_keys (pull_data, sign)) + continue; /* Set return to true if any sign fit */ if (ostree_sign_commit_verify (sign, From eb8e501ecef550191d60b53c7130988a3509e711 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 27 Oct 2019 21:21:21 +0300 Subject: [PATCH 038/177] lib/sign: minor optimisation for ed25519 Exclude unneeded conversion while load keys from files. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index f90a310c..662521b9 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -405,10 +405,7 @@ _load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError ** /* Read the key itself */ /* base64 encoded key */ - gsize key_len = 0; - g_autofree guchar *key = g_base64_decode (line, &key_len); - - pk = g_variant_new_fixed_array (G_VARIANT_TYPE_BYTE, key, key_len, sizeof(guchar)); + pk = g_variant_new_string (line); if (ostree_sign_ed25519_add_pk (self, pk, error)) { ret = TRUE; From ceaf6d7f546690601df98ee5e9d971f5b31096a9 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 27 Oct 2019 19:45:48 +0000 Subject: [PATCH 039/177] lib/sign: add ostree_seign_clear_keys function Add the function for implicit cleanup of all loaded keys. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-sign-ed25519.c | 40 ++++++++++++++++++++++++----- src/libostree/ostree-sign-ed25519.h | 3 +++ src/libostree/ostree-sign.c | 11 ++++++++ src/libostree/ostree-sign.h | 6 +++++ 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 440338c2..9b71d610 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -719,6 +719,7 @@ ostree_sign_data_verify ostree_sign_get_by_name ostree_sign_get_name ostree_sign_add_pk +ostree_sign_clear_keys ostree_sign_load_pk ostree_sign_set_pk ostree_sign_set_sk diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 3ca8f2c6..a10ec266 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -30,6 +30,7 @@ global: ostree_sign_data_verify; ostree_sign_get_by_name; ostree_sign_get_name; + ostree_sign_clear_keys; ostree_sign_load_pk; ostree_sign_set_pk; ostree_sign_add_pk; diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 662521b9..f61c3bdd 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -62,6 +62,7 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) self->get_name = ostree_sign_ed25519_get_name; self->metadata_key = ostree_sign_ed25519_metadata_key; self->metadata_format = ostree_sign_ed25519_metadata_format; + self->clear_keys = ostree_sign_ed25519_clear_keys; self->set_sk = ostree_sign_ed25519_set_sk; self->set_pk = ostree_sign_ed25519_set_pk; self->add_pk = ostree_sign_ed25519_add_pk; @@ -253,6 +254,36 @@ const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) return OSTREE_SIGN_METADATA_ED25519_TYPE; } +gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + +#ifdef HAVE_LIBSODIUM + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + /* Clear secret key */ + if (sign->secret_key != NULL) + { + memset (sign->secret_key, 0, crypto_sign_SECRETKEYBYTES); + g_free (sign->secret_key); + sign->secret_key = NULL; + } + + /* Clear already loaded trusted keys */ + if (sign->public_keys != NULL) + { + g_list_free_full (sign->public_keys, g_free); + sign->public_keys = NULL; + } + + return TRUE; + +#endif /* HAVE_LIBSODIUM */ + return FALSE; +} + /* Support 2 representations: * base64 ascii -- secret key is passed as string * raw key -- key is passed as bytes array @@ -267,7 +298,7 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - g_free (sign->secret_key); + ostree_sign_ed25519_clear_keys (self, error); gsize n_elements = 0; @@ -315,12 +346,7 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - /* Substitute the key(s) with a new one */ - if (sign->public_keys != NULL) - { - g_list_free_full (sign->public_keys, g_free); - sign->public_keys = NULL; - } + ostree_sign_ed25519_clear_keys (self, error); return ostree_sign_ed25519_add_pk (self, public_key, error); } diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index 4519961d..bced1cdf 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -55,6 +55,9 @@ const gchar * ostree_sign_ed25519_get_name (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self); const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self); +gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, + GError **error); + gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error); diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index b1975215..e5d55ef2 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -87,6 +87,17 @@ const gchar * ostree_sign_metadata_format (OstreeSign *self) return OSTREE_SIGN_GET_IFACE (self)->metadata_format (self); } +gboolean ostree_sign_clear_keys (OstreeSign *self, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + if (OSTREE_SIGN_GET_IFACE (self)->clear_keys == NULL) + return TRUE; + + return OSTREE_SIGN_GET_IFACE (self)->clear_keys (self, error); +} + gboolean ostree_sign_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 008c3f9d..9add0450 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -59,6 +59,8 @@ struct _OstreeSignInterface GError **error); const gchar *(* metadata_key) (OstreeSign *self); const gchar *(* metadata_format) (OstreeSign *self); + gboolean (* clear_keys) (OstreeSign *self, + GError **error); gboolean (* set_sk) (OstreeSign *self, GVariant *secret_key, GError **error); @@ -109,6 +111,10 @@ gboolean ostree_sign_commit_verify (OstreeSign *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sign_clear_keys (OstreeSign *self, + GError **error); + _OSTREE_PUBLIC gboolean ostree_sign_set_sk (OstreeSign *self, GVariant *secret_key, From ee12b7e7743d695c06c1ffbe1d76bcb53975f83b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 27 Oct 2019 23:15:10 +0300 Subject: [PATCH 040/177] lib/sign: add revoking mechanism for ed25519 keys Skip public keys verification if key is marked as invalid key. Allow to redefine system-wide directories for ed25519 verification. Minor bugfixes. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 258 +++++++++++++++++++++------- 1 file changed, 196 insertions(+), 62 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index f61c3bdd..0c7cd951 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -44,6 +44,7 @@ struct _OstreeSignEd25519 gboolean initialized; guchar *secret_key; GList *public_keys; + GList *revoked_keys; }; static void @@ -83,6 +84,7 @@ ostree_sign_ed25519_init (OstreeSignEd25519 *self) self->initialized = TRUE; self->secret_key = NULL; self->public_keys = NULL; + self->revoked_keys = NULL; #ifdef HAVE_LIBSODIUM if (sodium_init() < 0) @@ -139,6 +141,13 @@ err: return FALSE; } +#ifdef HAVE_LIBSODIUM +static gint +_compare_ed25519_keys(gconstpointer a, gconstpointer b) { + return memcmp (a, b, crypto_sign_PUBLICKEYBYTES); +} +#endif + gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, @@ -204,6 +213,15 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, public_key != NULL; public_key = public_key->next) { + + /* TODO: use non-list for tons of revoked keys? */ + if (g_list_find_custom (sign->revoked_keys, public_key->data, _compare_ed25519_keys) != NULL) + { + g_debug("Skip revoked key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + continue; + } + if (crypto_sign_verify_detached ((guchar *) g_variant_get_data (child), g_bytes_get_data (data, NULL), g_bytes_get_size (data), @@ -278,6 +296,13 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, sign->public_keys = NULL; } + /* Clear already loaded revoked keys */ + if (sign->revoked_keys != NULL) + { + g_list_free_full (sign->revoked_keys, g_free); + sign->revoked_keys = NULL; + } + return TRUE; #endif /* HAVE_LIBSODIUM */ @@ -344,8 +369,6 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - ostree_sign_ed25519_clear_keys (self, error); return ostree_sign_ed25519_add_pk (self, public_key, error); @@ -365,7 +388,7 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); g_autofree char * hex = NULL; - gpointer key = NULL; + gpointer key = NULL; gsize n_elements = 0; @@ -395,7 +418,7 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, goto err; } - if (g_list_find (sign->public_keys, key) == NULL) + if (g_list_find_custom (sign->public_keys, key, _compare_ed25519_keys) == NULL) { gpointer newkey = g_memdup (key, n_elements); sign->public_keys = g_list_prepend (sign->public_keys, newkey); @@ -408,10 +431,65 @@ err: return FALSE; } +#ifdef HAVE_LIBSODIUM +/* Add revoked public key */ +static gboolean +_ed25519_add_revoked (OstreeSign *self, + GVariant *revoked_key, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + g_autofree char * hex = NULL; + gpointer key = NULL; + + gsize n_elements = 0; + + if (g_variant_is_of_type (revoked_key, G_VARIANT_TYPE_STRING)) + { + const gchar *rk_ascii = g_variant_get_string (revoked_key, NULL); + key = g_base64_decode (rk_ascii, &n_elements); + } + else + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unknown ed25519 revoked key type"); + goto err; + } + + hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_debug ("Read ed25519 revoked key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); + + if (n_elements != crypto_sign_PUBLICKEYBYTES) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Incorrect ed25519 revoked key"); + goto err; + } + + if (g_list_find_custom (sign->revoked_keys, key, _compare_ed25519_keys) == NULL) + { + gpointer newkey = g_memdup (key, n_elements); + sign->revoked_keys = g_list_prepend (sign->revoked_keys, newkey); + } + + return TRUE; + +err: + return FALSE; +} +#endif /* HAVE_LIBSODIUM */ + static gboolean -_load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError **error) +_load_pk_from_stream (OstreeSign *self, + GDataInputStream *key_data_in, + gboolean trusted, + GError **error) { + g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (key_data_in, FALSE); #ifdef HAVE_LIBSODIUM gboolean ret = FALSE; @@ -422,23 +500,31 @@ _load_pk_from_stream (OstreeSign *self, GDataInputStream *key_data_in, GError ** gsize len = 0; g_autofree char *line = g_data_input_stream_read_line (key_data_in, &len, NULL, error); g_autoptr (GVariant) pk = NULL; + gboolean added = FALSE; if (*error != NULL) goto err; if (line == NULL) - goto out; + goto out; /* Read the key itself */ /* base64 encoded key */ pk = g_variant_new_string (line); - if (ostree_sign_ed25519_add_pk (self, pk, error)) - { - ret = TRUE; - g_debug ("Added public key: %s", line); - } + + if (trusted) + added = ostree_sign_ed25519_add_pk (self, pk, error); else - g_debug ("Invalid public key: %s", line); + added = _ed25519_add_revoked (self, pk, error); + + g_debug ("%s %s key: %s", + added ? "Added" : "Invalid", + trusted ? "public" : "revoked", + line); + + /* Mark what we load at least one key */ + if (added) + ret = TRUE; } out: @@ -452,6 +538,7 @@ err: static gboolean _load_pk_from_file (OstreeSign *self, const gchar *filename, + gboolean trusted, GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -477,64 +564,63 @@ _load_pk_from_file (OstreeSign *self, key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); g_assert (key_data_in != NULL); - if (!_load_pk_from_stream (self, key_data_in, error)) - goto err; + if (!_load_pk_from_stream (self, key_data_in, trusted, error)) + { + if (error == NULL || *error == NULL) + g_set_error (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: ed25519: no valid keys in file '%s'", + filename); + goto err; + } return TRUE; err: return FALSE; } -gboolean -ostree_sign_ed25519_load_pk (OstreeSign *self, - GVariant *options, - GError **error) +static gboolean +_ed25519_load_pk (OstreeSign *self, + GVariant *options, + gboolean trusted, + GError **error) { g_debug ("%s enter", __FUNCTION__); gboolean ret = FALSE; + const gchar *custom_dir = NULL; - /* Default paths there to find files with public keys */ - const gchar *default_dirs[] = + g_autoptr (GPtrArray) base_dirs = g_ptr_array_new_with_free_func (g_free); + g_autoptr (GPtrArray) ed25519_files = g_ptr_array_new_with_free_func (g_free); + + if (g_variant_lookup (options, "basedir", "&s", &custom_dir)) { - "/etc/ostree/trusted.ed25519.d", - DATADIR "/ostree/trusted.ed25519.d" - }; - const gchar *default_files[] = + /* Add custom directory */ + g_ptr_array_add (base_dirs, g_strdup (custom_dir)); + } + else { - "/etc/ostree/trusted.ed25519", - DATADIR "/ostree/trusted.ed25519" - }; - - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - - const gchar *filename = NULL; - - /* Clear already loaded keys */ - if (sign->public_keys != NULL) - { - g_list_free_full (sign->public_keys, g_free); - sign->public_keys = NULL; + /* Default paths where to find files with public keys */ + g_ptr_array_add (base_dirs, g_strdup ("/etc/ostree")); + g_ptr_array_add (base_dirs, g_strdup (DATADIR "/ostree")); } - /* Read only file provided */ - if (g_variant_lookup (options, "filename", "&s", &filename)) - return _load_pk_from_file (self, filename, error); - - /* Scan all well-known files and directories */ - for (gint i=0; i < G_N_ELEMENTS(default_files); i++) - if (!_load_pk_from_file (self, default_files[i], error)) - { - g_debug ("Problem with loading ed25519 public keys from `%s`", default_files[i]); - g_clear_error(error); - } - else - ret = TRUE; - - /* Scan all well-known files and directories */ - for (gint i=0; i < G_N_ELEMENTS(default_dirs); i++) + /* Scan all well-known directories and construct the list with file names to scan keys */ + for (gint i=0; i < base_dirs->len; i++) { - g_autoptr (GDir) dir = g_dir_open (default_dirs[i], 0, error); + gchar *base_name = NULL; + g_autofree gchar *base_dir = NULL; + g_autoptr (GDir) dir = NULL; + + base_name = g_build_filename ((gchar *)g_ptr_array_index (base_dirs, i), + trusted ? "trusted.ed25519" : "revoked.ed25519", + NULL); + + g_debug ("Check ed25519 keys from file: %s", base_name); + g_ptr_array_add (ed25519_files, base_name); + + base_dir = g_strconcat (base_name, ".d", NULL); + dir = g_dir_open (base_dir, 0, error); if (dir == NULL) { g_clear_error (error); @@ -543,17 +629,65 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, const gchar *entry = NULL; while ((entry = g_dir_read_name (dir)) != NULL) { - filename = g_build_filename (default_dirs[i], entry, NULL); - if (!_load_pk_from_file (self, filename, error)) - { - g_debug ("Problem with loading ed25519 public keys from `%s`", filename); - g_clear_error(error); - } - else - ret = TRUE; + gchar *filename = g_build_filename (base_dir, entry, NULL); + g_debug ("Check ed25519 keys from file: %s", filename); + g_ptr_array_add (ed25519_files, filename); } } + /* Scan all well-known files */ + for (gint i=0; i < ed25519_files->len; i++) + { + if (!_load_pk_from_file (self, (gchar *)g_ptr_array_index (ed25519_files, i), trusted, error)) + { + g_debug ("Problem with loading ed25519 %s keys from `%s`", + trusted ? "public" : "revoked", + (gchar *)g_ptr_array_index (ed25519_files, i)); + g_clear_error(error); + } + else + ret = TRUE; + } + + if (!ret && (error == NULL || *error == NULL)) + g_set_error_literal (error, + G_IO_ERROR, G_IO_ERROR_FAILED, + "signature: ed25519: no keys loaded"); + return ret; } +/* + * options argument should be a{sv}: + * - filename -- single file to use to load keys from; + * - basedir -- directory containing subdirectories + * 'trusted.ed25519.d' and 'revoked.ed25519.d' with appropriate + * public keys. Used for testing and re-definition of system-wide + * directories if defaults are not suitable for any reason. + */ +gboolean +ostree_sign_ed25519_load_pk (OstreeSign *self, + GVariant *options, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + const gchar *filename = NULL; + + /* Read keys only from single file provided */ + if (g_variant_lookup (options, "filename", "&s", &filename)) + return _load_pk_from_file (self, filename, TRUE, error); + + /* Load public keys from well-known directories and files */ + if (!_ed25519_load_pk (self, options, TRUE, error)) + return FALSE; + + /* Load untrusted keys from well-known directories and files + * Ignore the failure from this function -- it is expected to have + * empty list of revoked keys. + * */ + if (!_ed25519_load_pk (self, options, FALSE, error)) + g_clear_error(error); + + return TRUE; +} From 200efd7d44cfc490d2bbca7e09d494e851e6c890 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 29 Oct 2019 22:16:09 +0300 Subject: [PATCH 041/177] builtin/sign: add option 'keys-dir' Option '--keys-dir' is used for redefinition of default directories with public/revoked keys. If keys directory is set then default directories are ignored and target directory is expected to contain following structure for ed25519 signature mechanism: dir/ trusted.ed25519 <- file with trusted keys revoked.ed25519 <- file with revoked keys trusted.ed25519.d/ <- directory with files containing trusted keys revoked.ed25519.d/ <- directory with files containing revoked keys Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-sign.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index f673631d..73561b43 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -37,6 +37,7 @@ static gboolean opt_delete; static gboolean opt_verify; static char *opt_sign_name; static char *opt_filename; +static char *opt_keysdir; /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -48,9 +49,10 @@ static GOptionEntry options[] = { { "verify", 0, 0, G_OPTION_ARG_NONE, &opt_verify, "Verify signatures", NULL}, { "sign-type", 's', 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, #if defined(HAVE_LIBSODIUM) - { "keys-file", 's', 0, G_OPTION_ARG_STRING, &opt_filename, "Read key(s) from file", "NAME"}, + { "keys-file", 0, 0, G_OPTION_ARG_STRING, &opt_filename, "Read key(s) from file", "NAME"}, + { "keys-dir", 0, 0, G_OPTION_ARG_STRING, &opt_keysdir, "Redefine system-wide directories with public and revoked keys for verification", "NAME"}, #endif - { NULL } + { NULL } }; static void @@ -131,7 +133,10 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, resolved_commit, cancellable, &local_error)) - ret = TRUE; + { + ret = TRUE; + goto out; + } } else { @@ -162,6 +167,9 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, g_autoptr (GVariant) options = NULL; builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + /* Use custom directory with public and revoked keys instead of system-wide directories */ + if (opt_keysdir) + g_variant_builder_add (builder, "{sv}", "basedir", g_variant_new_string (opt_keysdir)); /* The last chance for verification source -- system files */ if (opt_filename) g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (opt_filename)); @@ -235,9 +243,8 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, } } } - // No valid signature found - if (opt_verify && (ret != TRUE)) + if (opt_verify && (ret != TRUE) && (*error == NULL)) g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "No valid signatures found"); From 7e71f87ebc11fb930e7c42d8db263b1fe6073e98 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 29 Oct 2019 22:23:55 +0300 Subject: [PATCH 042/177] tests/sign: check system-wide config and revoked keys Extend the ed25519 tests with checking the system-wide directory keys loading code(with the help of redefinition). Added test of ed25519 revoking keys mechanism. Signed-off-by: Denis Pynkin --- tests/test-signed-commit.sh | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index e4138817..c523aedd 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..8" +echo "1..10" mkdir ${test_tmpdir}/repo ostree_repo_init repo --mode="archive" @@ -58,6 +58,8 @@ if ! has_libsodium; then echo "ok multiple signing # SKIP due libsodium unavailability" echo "ok verify ed25519 keys file # SKIP due libsodium unavailability" echo "ok sign with ed25519 keys file # SKIP due libsodium unavailability" + echo "ok verify ed25519 system-wide configuration # SKIP due libsodium unavailability" + echo "ok verify ed25519 revoking keys mechanism # SKIP due libsodium unavailability" exit 0 fi @@ -125,10 +127,12 @@ PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then exit 1 fi + # Test if have a problem with file object if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${test_tmpdir} ${COMMIT}; then exit 1 fi + # Test with single key in list echo ${PUBLIC} > ${PUBKEYS} ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} @@ -169,3 +173,27 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 --keys- ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} echo "ok sign with ed25519 keys file" +# Check the well-known places mechanism +mkdir -p ${test_tmpdir}/{trusted,revoked}.ed25519.d +for((i=0;i<100;i++)); do + # Generate some key files with random public signatures + openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 > ${test_tmpdir}/trusted.ed25519.d/signature_$i +done +# Check no valid public keys are available +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-dir=${test_tmpdir} ${COMMIT}; then + exit 1 +fi +echo ${PUBLIC} > ${test_tmpdir}/trusted.ed25519.d/correct +# Verify with correct key +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-dir=${test_tmpdir} ${COMMIT} + +echo "ok verify ed25519 system-wide configuration" + +# Add the public key into revoked list +echo ${PUBLIC} > ${test_tmpdir}/revoked.ed25519.d/correct +# Check if public key is not valid anymore +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-dir=${test_tmpdir} ${COMMIT}; then + exit 1 +fi +rm -rf ${test_tmpdir}/{trusted,revoked}.ed25519.d +echo "ok verify ed25519 revoking keys mechanism" From 0c89055b191a0dd90489418402a51c20e5f1f935 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 1 Nov 2019 02:44:25 +0300 Subject: [PATCH 043/177] man: document `ostree sign` Add man page for `ostree sign`. Signed-off-by: Denis Pynkin --- Makefile-man.am | 2 +- man/ostree-sign.xml | 152 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 man/ostree-sign.xml diff --git a/Makefile-man.am b/Makefile-man.am index bc58103b..718e773c 100644 --- a/Makefile-man.am +++ b/Makefile-man.am @@ -32,7 +32,7 @@ ostree-commit.1 ostree-create-usb.1 ostree-export.1 \ ostree-config.1 ostree-diff.1 ostree-find-remotes.1 ostree-fsck.1 \ ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 \ ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 \ -ostree-rev-parse.1 ostree-show.1 ostree-summary.1 \ +ostree-rev-parse.1 ostree-show.1 ostree-sign.1 ostree-summary.1 \ ostree-static-delta.1 if USE_LIBSOUP man1_files += ostree-trivial-httpd.1 diff --git a/man/ostree-sign.xml b/man/ostree-sign.xml new file mode 100644 index 00000000..50c0b337 --- /dev/null +++ b/man/ostree-sign.xml @@ -0,0 +1,152 @@ + + + + + + + + + ostree sign + OSTree + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + ostree sign + 1 + + + + ostree-sign + Sign a commit + + + + + ostree sign OPTIONS COMMIT KEY-ID + + + + + Description + + + Add a new signature to a commit. + + Note that currently, this will append a new signature even if + the commit is already signed with a given key. + + + + There are several "well-known" system places for `ed25519` trusted and revoked public keys -- expected single base64-encoded key per line. + + + Files: + + /etc/ostree/trusted.ed25519 + /etc/ostree/revoked.ed25519 + /usr/share/ostree/trusted.ed25519 + /usr/share/ostree/revoked.ed25519 + + + + Directories containing files with keys: + + /etc/ostree/trusted.ed25519.d + /etc/ostree/revoked.ed25519.d + /usr/share/ostree/trusted.ed25519.d + /usr/share/ostree/rvokeded.ed25519.d + + + + + + Options + + + + + + + + + + base64-encoded secret (for signing) or public key (for verifying). + + + + + + + ASCII-string used as secret key and public key. + + + + + + + + + Verify signatures + + + + + + Use particular signature mechanism. Currently + available ed25519 and dummy + signature types. + + The default is ed25519. + + + + + + Read key(s) from file filename. + + + + Valid for ed25519 signature type. + For ed25519 this file must contain base64-encoded + secret key(s) (for signing) or public key(s) (for verifying) per line. + + + + + + Redefine the system path, where to search files and subdirectories with + well-known and revoked keys. + + + + + From e9b1ebfc1152ca49106021a08865da89acfc1512 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 1 Nov 2019 03:00:13 +0300 Subject: [PATCH 044/177] bash-completion: add completion for `ostree sign` Add bash completion with supported options for signing command. Signed-off-by: Denis Pynkin --- bash/ostree | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/bash/ostree b/bash/ostree index 4aec588b..7256e40a 100644 --- a/bash/ostree +++ b/bash/ostree @@ -1484,6 +1484,48 @@ _ostree_show() { return 0 } +_ostree_sign() { + local boolean_options=" + $main_boolean_options + --delete -d + --verify -v + " + + local options_with_args=" + --sign-type + --keys-file + --keys-dir + --repo + " + + local options_with_args_glob=$( __ostree_to_extglob "$options_with_args" ) + + case "$prev" in + --keys-file|--keys-dir|--repo) + __ostree_compreply_dirs_only + return 0 + ;; + $options_with_args_glob ) + return 0 + ;; + esac + + case "$cur" in + -*) + local all_options="$boolean_options $options_with_args" + __ostree_compreply_all_options + ;; + *) + local argpos=$( __ostree_pos_first_nonflag $( __ostree_to_alternatives "$options_with_args" ) ) + + if [ $cword -eq $argpos ]; then + __ostree_compreply_commits + fi + esac + + return 0 +} + _ostree_static_delta_apply_offline() { local boolean_options=" $main_boolean_options @@ -1747,6 +1789,7 @@ _ostree() { reset rev-parse show + sign static-delta summary " From 908a2cd7601491195db6c4e6151a601b002c25e0 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 10 Nov 2019 16:51:23 +0300 Subject: [PATCH 045/177] apidoc: add API documentation for signing interface Add the documentation for all public functions. Signed-off-by: Denis Pynkin --- apidoc/ostree-docs.xml | 1 + src/libostree/ostree-sign.c | 355 +++++++++++++++++++++++++++++------- src/libostree/ostree-sign.h | 30 +-- 3 files changed, 298 insertions(+), 88 deletions(-) diff --git a/apidoc/ostree-docs.xml b/apidoc/ostree-docs.xml index 8721ffa8..1ad0de37 100644 --- a/apidoc/ostree-docs.xml +++ b/apidoc/ostree-docs.xml @@ -21,6 +21,7 @@ + diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index e5d55ef2..95319f67 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -22,6 +22,15 @@ * */ +/** + * SECTION:ostree-sign + * @title: Signature management + * @short_description: Sign and verify commits + * + * An #OstreeSign interface allows to select and use any available engine + * for signing or verifying the commit object or summary file. + */ + #include "config.h" #include @@ -71,7 +80,20 @@ ostree_sign_default_init (OstreeSignInterface *iface) g_debug ("OstreeSign initialization"); } -const gchar * ostree_sign_metadata_key (OstreeSign *self) +/** + * ostree_sign_metadata_key: + * @self: an #OstreeSign object + * + * Return the pointer to the name of the key used in (detached) metadata for + * current signing engine. + * + * Returns: (transfer none): pointer to the metadata key name, + * @NULL in case of error (unlikely). + * + * Since: 2020.2 + */ +const gchar * +ostree_sign_metadata_key (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); @@ -79,7 +101,20 @@ const gchar * ostree_sign_metadata_key (OstreeSign *self) return OSTREE_SIGN_GET_IFACE (self)->metadata_key (self); } -const gchar * ostree_sign_metadata_format (OstreeSign *self) +/** + * ostree_sign_metadata_format: + * @self: an #OstreeSign object + * + * Return the pointer to the string with format used in (detached) metadata for + * current signing engine. + * + * Returns: (transfer none): pointer to the metadata format, + * @NULL in case of error (unlikely). + * + * Since: 2020.2 + */ +const gchar * +ostree_sign_metadata_format (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); @@ -87,8 +122,20 @@ const gchar * ostree_sign_metadata_format (OstreeSign *self) return OSTREE_SIGN_GET_IFACE (self)->metadata_format (self); } -gboolean ostree_sign_clear_keys (OstreeSign *self, - GError **error) +/** + * ostree_sign_clear_keys: + * @self: an #OstreeSign object + * @error: a #GError + * + * Clear all previously preloaded secret and public keys. + * + * Returns: @TRUE in case if no errors, @FALSE in case of error + * + * Since: 2020.2 + */ +gboolean +ostree_sign_clear_keys (OstreeSign *self, + GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -98,9 +145,25 @@ gboolean ostree_sign_clear_keys (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->clear_keys (self, error); } -gboolean ostree_sign_set_sk (OstreeSign *self, - GVariant *secret_key, - GError **error) +/** + * ostree_sign_set_sk: + * @self: an #OstreeSign object + * @secret_key: secret key to be added + * @error: a #GError + * + * Set the secret key to be used for signing data, commits and summary. + * + * The @secret_key argument depends of the particular engine implementation. + * + * Returns: @TRUE in case if the key could be set successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 + */ +gboolean +ostree_sign_set_sk (OstreeSign *self, + GVariant *secret_key, + GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -110,9 +173,26 @@ gboolean ostree_sign_set_sk (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->set_sk (self, secret_key, error); } -gboolean ostree_sign_set_pk (OstreeSign *self, - GVariant *public_key, - GError **error) +/** + * ostree_sign_set_pk: + * @self: an #OstreeSign object + * @public_key: single public key to be added + * @error: a #GError + * + * Set the public key for verification. It is expected what all + * previously pre-loaded public keys will be dropped. + * + * The @public_key argument depends of the particular engine implementation. + * + * Returns: @TRUE in case if the key could be set successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 + */ +gboolean +ostree_sign_set_pk (OstreeSign *self, + GVariant *public_key, + GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -122,9 +202,26 @@ gboolean ostree_sign_set_pk (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->set_pk (self, public_key, error); } -gboolean ostree_sign_add_pk (OstreeSign *self, - GVariant *public_key, - GError **error) +/** + * ostree_sign_add_pk: + * @self: an #OstreeSign object + * @public_key: single public key to be added + * @error: a #GError + * + * Add the public key for verification. Could be called multiple times for + * adding all needed keys to be used for verification. + * + * The @public_key argument depends of the particular engine implementation. + * + * Returns: @TRUE in case if the key could be added successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 + */ +gboolean +ostree_sign_add_pk (OstreeSign *self, + GVariant *public_key, + GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -134,9 +231,33 @@ gboolean ostree_sign_add_pk (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->add_pk (self, public_key, error); } -/* Load private keys for verification from anywhere. - * No need to have the same function for secret keys -- the signing SW must do it in it's own way - * */ +/** + * ostree_sign_load_pk: + * @self: an #OstreeSign object + * @options: any options + * @error: a #GError + * + * Load public keys for verification from anywhere. + * It is expected that all keys would be added to already pre-loaded keys. + * + * The @options argument depends of the particular engine implementation. + * + * For example, @ed25515 engine could use following string-formatted options: + * - @filename -- single file to use to load keys from + * - @basedir -- directory containing subdirectories + * 'trusted.ed25519.d' and 'revoked.ed25519.d' with appropriate + * public keys. Used for testing and re-definition of system-wide + * directories if defaults are not suitable for any reason. + * + * Returns: @TRUE in case if at least one key could be load successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 + */ +/* + * No need to have similar function for secret keys load -- it is expected + * what the signing software will load the secret key in it's own way. + */ gboolean ostree_sign_load_pk (OstreeSign *self, GVariant *options, @@ -150,11 +271,30 @@ ostree_sign_load_pk (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, options, error); } -gboolean ostree_sign_data (OstreeSign *self, - GBytes *data, - GBytes **signature, - GCancellable *cancellable, - GError **error) +/** + * ostree_sign_data: + * @self: an #OstreeSign object + * @data: the raw data to be signed with pre-loaded secret key + * @signature: in case of success will contain signature + * @cancellable: A #GCancellable + * @error: a #GError + * + * Sign the given @data with pre-loaded secret key. + * + * Depending of the signing engine used you will need to load + * the secret key with #ostree_sign_set_sk. + * + * Returns: @TRUE if @data has been signed successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 + */ +gboolean +ostree_sign_data (OstreeSign *self, + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error) { g_debug ("%s enter", __FUNCTION__); @@ -164,11 +304,29 @@ gboolean ostree_sign_data (OstreeSign *self, return OSTREE_SIGN_GET_IFACE (self)->data (self, data, signature, cancellable, error); } +/** + * ostree_sign_data_verify: + * @self: an #OstreeSign object + * @data: the raw data to check + * @signatures: the signatures to be checked + * @error: a #GError + * + * Verify given data against signatures with pre-loaded public keys. + * + * Depending of the signing engine used you will need to load + * the public key(s) with #ostree_sign_set_pk, #ostree_sign_add_pk + * or #ostree_sign_load_pk. + * + * Returns: @TRUE if @data has been signed at least with any single valid key, + * @FALSE in case of error or no valid keys are available (@error will contain the reason). + * + * Since: 2020.2 + */ gboolean ostree_sign_data_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error) + GBytes *data, + GVariant *signatures, + GError **error) { g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -213,6 +371,25 @@ _sign_detached_metadata_append (OstreeSign *self, return g_variant_dict_end (&metadata_dict); } +/** + * ostree_sign_commit_verify: + * @self: an #OstreeSign object + * @repo: an #OsreeRepo object + * @commit_checksum: SHA256 of given commit to verify + * @cancellable: A #GCancellable + * @error: a #GError + * + * Verify if commit is signed with known key. + * + * Depending of the signing engine used you will need to load + * the public key(s) for verification with #ostree_sign_set_pk, + * #ostree_sign_add_pk and/or #ostree_sign_load_pk. + * + * Returns: @TRUE if commit has been verified successfully, + * @FALSE in case of error or no valid keys are available (@error will contain the reason). + * + * Since: 2020.2 + */ gboolean ostree_sign_commit_verify (OstreeSign *self, OstreeRepo *repo, @@ -254,60 +431,51 @@ ostree_sign_commit_verify (OstreeSign *self, return ostree_sign_data_verify (self, - signed_data, - signatures, - error); + signed_data, + signatures, + error); } -const gchar * ostree_sign_get_name (OstreeSign *self) +/** + * ostree_sign_get_name: + * @self: an #OstreeSign object + * + * Return the pointer to the name of currently used/selected signing engine. + * + * The list of available engines could be acquired with #ostree_sign_list_names. + * + * Returns: (transfer none): pointer to the name + * @NULL in case of error (unlikely). + * + * Since: 2020.2 + */ +const gchar * +ostree_sign_get_name (OstreeSign *self) { g_debug ("%s enter", __FUNCTION__); - g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->get_name != NULL, FALSE); + g_return_val_if_fail (OSTREE_IS_SIGN (self), NULL); + g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->get_name != NULL, NULL); return OSTREE_SIGN_GET_IFACE (self)->get_name (self); } -OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error) -{ - g_debug ("%s enter", __FUNCTION__); - - OstreeSign *sign = NULL; - - /* Get types if not initialized yet */ -#if defined(HAVE_LIBSODIUM) - if (sign_types[SIGN_ED25519].type == 0) - sign_types[SIGN_ED25519].type = OSTREE_TYPE_SIGN_ED25519; -#endif - if (sign_types[SIGN_DUMMY].type == 0) - sign_types[SIGN_DUMMY].type = OSTREE_TYPE_SIGN_DUMMY; - - for (gint i=0; i < G_N_ELEMENTS(sign_types); i++) - { - if (g_strcmp0 (name, sign_types[i].name) == 0) - { - g_debug ("Found '%s' signing module", sign_types[i].name); - sign = g_object_new (sign_types[i].type, NULL); - break; - } - } - - if (sign == NULL) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Requested signature type is not implemented"); - - return sign; -} - - /** * ostree_sign_commit: - * @self: Self + * @self: an #OstreeSign object + * @repo: an #OsreeRepo object * @commit_checksum: SHA256 of given commit to sign * @cancellable: A #GCancellable * @error: a #GError * - * Add a GPG signature to a commit. + * Add a signature to a commit. + * + * Depending of the signing engine used you will need to load + * the secret key with #ostree_sign_set_sk. + * + * Returns: @TRUE if commit has been signed successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.2 */ gboolean ostree_sign_commit (OstreeSign *self, @@ -354,7 +522,17 @@ ostree_sign_commit (OstreeSign *self, return TRUE; } -GStrv ostree_sign_list_names(void) +/** + * ostree_sign_list_names: + * + * Return an array with all available sign engines names. + * + * Returns: (transfer full): an array of strings, free when you used it + * + * Since: 2020.2 + */ +GStrv +ostree_sign_list_names(void) { g_debug ("%s enter", __FUNCTION__); @@ -364,8 +542,53 @@ GStrv ostree_sign_list_names(void) for (i=0; i < G_N_ELEMENTS(sign_types); i++) { names[i] = g_strdup(sign_types[i].name); - g_debug ("Found '%s' signing module", names[i]); + g_debug ("Found '%s' signing engine", names[i]); } return names; } + +/** + * ostree_sign_get_by_name: + * @name: the name of desired signature engine + * @error: return location for a #GError + * + * Tries to find and return proper signing engine by it's name. + * + * The list of available engines could be acquired with #ostree_sign_list_names. + * + * Returns: (transfer full): a constant, free when you used it + * + * Since: 2020.2 + */ +OstreeSign * +ostree_sign_get_by_name (const gchar *name, GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + OstreeSign *sign = NULL; + + /* Get types if not initialized yet */ +#if defined(HAVE_LIBSODIUM) + if (sign_types[SIGN_ED25519].type == 0) + sign_types[SIGN_ED25519].type = OSTREE_TYPE_SIGN_ED25519; +#endif + if (sign_types[SIGN_DUMMY].type == 0) + sign_types[SIGN_DUMMY].type = OSTREE_TYPE_SIGN_DUMMY; + + for (gint i=0; i < G_N_ELEMENTS(sign_types); i++) + { + if (g_strcmp0 (name, sign_types[i].name) == 0) + { + g_debug ("Using '%s' signing engine", sign_types[i].name); + sign = g_object_new (sign_types[i].type, NULL); + break; + } + } + + if (sign == NULL) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Requested signature type is not implemented"); + + return sign; +} diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 9add0450..87ed25ce 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -55,7 +55,7 @@ struct _OstreeSignInterface GError **error); gboolean (* data_verify) (OstreeSign *self, GBytes *data, - GVariant *metadata, + GVariant *signatures, GError **error); const gchar *(* metadata_key) (OstreeSign *self); const gchar *(* metadata_format) (OstreeSign *self); @@ -80,16 +80,16 @@ const gchar * ostree_sign_get_name (OstreeSign *self); _OSTREE_PUBLIC gboolean ostree_sign_data (OstreeSign *self, - GBytes *data, - GBytes **signature, - GCancellable *cancellable, - GError **error); + GBytes *data, + GBytes **signature, + GCancellable *cancellable, + GError **error); _OSTREE_PUBLIC gboolean ostree_sign_data_verify (OstreeSign *self, - GBytes *data, - GVariant *signatures, - GError **error); + GBytes *data, + GVariant *signatures, + GError **error); _OSTREE_PUBLIC const gchar * ostree_sign_metadata_key (OstreeSign *self); @@ -136,23 +136,9 @@ gboolean ostree_sign_load_pk (OstreeSign *self, GError **error); -/** - * ostree_sign_list_names: - * - * Return the array with all available sign modules names. - * - * Returns: (transfer full): an array of strings, free when you used it - */ _OSTREE_PUBLIC GStrv ostree_sign_list_names(void); -/** - * ostree_sign_get_by_name: - * - * Tries to find and return proper signing engine by it's name. - * - * Returns: (transfer full): a constant, free when you used it - */ _OSTREE_PUBLIC OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error); From e799186658e868f1956942ab3d88fd54fd5d441e Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 10 Nov 2019 19:17:58 +0300 Subject: [PATCH 046/177] man: document commit signing Added options descriptions for `ostree-commit` allowing to sign the commit. Signed-off-by: Denis Pynkin --- man/ostree-commit.xml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index c64a7a00..2c821fc1 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -251,6 +251,39 @@ Boston, MA 02111-1307, USA. POLICY is a boolean which specifies whether fsync should be used or not. Default to true. + + + + + Use particular signature engine. Currently + available ed25519 and dummy + signature types. + + The default is ed25519. + + + + + ="KEY-ID" + + There KEY-ID is: + + + + + base64-encoded secret key for commit signing. + + + + + + + ASCII-string used as secret key. + + + + + From 2fd94388b129724b1bc78fcd280a9680cb16ca01 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 19:58:29 +0300 Subject: [PATCH 047/177] bin/pull-local: add --sign-verify Add option for enabling verification while pulling from local. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-pull-local.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 4b3224f3..4dbd3bfd 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -39,6 +39,7 @@ static gboolean opt_bareuseronly_files; static gboolean opt_require_static_deltas; static gboolean opt_gpg_verify; static gboolean opt_gpg_verify_summary; +static gboolean opt_sign_verify; static int opt_depth = 0; /* ATTENTION: @@ -55,6 +56,7 @@ static GOptionEntry options[] = { { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify, "GPG verify commits (must specify --remote)", NULL }, { "gpg-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify_summary, "GPG verify summary (must specify --remote)", NULL }, + { "sign-verify", 0, 0, G_OPTION_ARG_NONE, &opt_sign_verify, "Verify commits signature (must specify --remote)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -182,6 +184,10 @@ ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invoc g_variant_builder_add (&builder, "{s@v}", "depth", g_variant_new_variant (g_variant_new_int32 (opt_depth))); + if (opt_sign_verify) + g_variant_builder_add (&builder, "{s@v}", "sign-verify", + g_variant_new_variant (g_variant_new_boolean (TRUE))); + if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); else From a9df634c474033b29640f57454d1b73fc5f1a070 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 20:02:13 +0300 Subject: [PATCH 048/177] tests/libtest: add functions for ed25519 tests Add functions for keys generation to be used in signing-related tests: - gen_ed25519_keys initializing variables ED25519PUBLIC, ED25519SEED and ED25519SECRET with appropriate base64-encoded keys - gen_ed25519_random_public print a random base64 public key (used in tests with wrong keys) Signed-off-by: Denis Pynkin --- tests/libtest.sh | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 58a9fd9b..c473fd82 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -673,6 +673,12 @@ which_gpg () { echo ${gpg} } +libtest_cleanup_gpg () { + local gpg_homedir=${1:-${test_tmpdir}/gpghome} + gpg-connect-agent --homedir "${gpg_homedir}" killagent /bye || true +} +libtest_exit_cmds+=(libtest_cleanup_gpg) + has_libsodium () { local ret ${CMD_PREFIX} ostree --version > version.txt @@ -682,12 +688,33 @@ has_libsodium () { return ${ret} } +# Keys for ed25519 signing tests +ED25519PUBLIC= +ED25519SEED= +ED25519SECRET= -libtest_cleanup_gpg () { - local gpg_homedir=${1:-${test_tmpdir}/gpghome} - gpg-connect-agent --homedir "${gpg_homedir}" killagent /bye || true +gen_ed25519_keys () +{ + # Generate private key in PEM format + pemfile="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" + openssl genpkey -algorithm ed25519 -outform PEM -out "${pemfile}" + + # Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html + # Extract the private and public parts from generated key. + ED25519PUBLIC="$(openssl pkey -outform DER -pubout -in ${pemfile} | tail -c 32 | base64)" + ED25519SEED="$(openssl pkey -outform DER -in ${pemfile} | tail -c 32 | base64)" + # Secret key is concantination of SEED and PUBLIC + ED25519SECRET="$(echo ${ED25519SEED}${ED25519PUBLIC} | base64 -d | base64 -w 0)" + + echo "Generated ed25519 keys:" + echo "public: ${ED25519PUBLIC}" + echo " seed: ${ED25519SEED}" +} + +gen_ed25519_random_public() +{ + openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 } -libtest_exit_cmds+=(libtest_cleanup_gpg) is_bare_user_only_repo () { grep -q 'mode=bare-user-only' $1/config From e474033ea99a89be4ef1984435794409509fdbf7 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 20:15:46 +0300 Subject: [PATCH 049/177] tests/sign: use library functions for ed25519 keys Switch to library functions usage. Signed-off-by: Denis Pynkin --- tests/test-signed-commit.sh | 28 +++++++++++----------------- tests/test-signed-pull.sh | 19 +++++-------------- 2 files changed, 16 insertions(+), 31 deletions(-) diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index c523aedd..6730a6df 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -64,18 +64,12 @@ if ! has_libsodium; then fi # Test ostree sign with 'ed25519' module -# Generate private key in PEM format -PEMFILE="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" -openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" +gen_ed25519_keys +PUBLIC=${ED25519PUBLIC} +SEED=${ED25519SEED} +SECRET=${ED25519SECRET} -# Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html -# Extract the private and public parts from generated key. -PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" -SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" -# Secret key is concantination of SEED and PUBLIC -SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" - -WRONG_PUBLIC="$(openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64)" +WRONG_PUBLIC="$(gen_ed25519_random_public)" echo "SEED = $SEED" echo "PUBLIC = $PUBLIC" @@ -94,10 +88,10 @@ if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed fi ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} ${PUBLIC} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${PUBLIC} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${WRONG_PUBLIC} ${PUBLIC} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} ${WRONG_PUBLIC} ${WRONG_PUBLIC} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${WRONG_PUBLIC} ${WRONG_PUBLIC} ${PUBLIC} ${WRONG_PUBLIC} ${WRONG_PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} $(gen_ed25519_random_public) ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} $(gen_ed25519_random_public) $(gen_ed25519_random_public) ${PUBLIC} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} $(gen_ed25519_random_public) $(gen_ed25519_random_public) +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} $(gen_ed25519_random_public) $(gen_ed25519_random_public) ${PUBLIC} $(gen_ed25519_random_public) $(gen_ed25519_random_public) echo "ok ed25519 signature verified" # Check if we able to use all available modules to sign the same commit @@ -140,7 +134,7 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed2551 # Test the file with multiple keys without a valid public key for((i=0;i<100;i++)); do # Generate a list with some public signatures - openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 + gen_ed25519_random_public done > ${PUBKEYS} # Check if file contain no valid signatures if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then @@ -177,7 +171,7 @@ echo "ok sign with ed25519 keys file" mkdir -p ${test_tmpdir}/{trusted,revoked}.ed25519.d for((i=0;i<100;i++)); do # Generate some key files with random public signatures - openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 > ${test_tmpdir}/trusted.ed25519.d/signature_$i + gen_ed25519_random_public done # Check no valid public keys are available if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-dir=${test_tmpdir} ${COMMIT}; then diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index dc922e81..238ce8e0 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -70,19 +70,10 @@ test_signed_pull "dummy" # Test ostree sign with 'ed25519' module -# Generate private key in PEM format -PEMFILE="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.pem)" -openssl genpkey -algorithm ed25519 -outform PEM -out "${PEMFILE}" - -# Based on: http://openssl.6102.n7.nabble.com/ed25519-key-generation-td73907.html -# Extract the private and public parts from generated key. -PUBLIC="$(openssl pkey -outform DER -pubout -in ${PEMFILE} | tail -c 32 | base64)" -SEED="$(openssl pkey -outform DER -in ${PEMFILE} | tail -c 32 | base64)" -# Secret key is concantination of SEED and PUBLIC -SECRET="$(echo ${SEED}${PUBLIC} | base64 -d | base64 -w 0)" - -echo "SEED = $SEED" -echo "PUBLIC = $PUBLIC" +gen_ed25519_keys +PUBLIC=${ED25519PUBLIC} +SEED=${ED25519SEED} +SECRET=${ED25519SECRET} COMMIT_ARGS="--sign=${SECRET} --sign-type=ed25519" @@ -96,7 +87,7 @@ PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" # Test the file with multiple keys without a valid public key for((i=0;i<100;i++)); do # Generate a list with some public signatures - openssl genpkey -algorithm ED25519 | openssl pkey -outform DER | tail -c 32 | base64 + gen_ed25519_random_public done > ${PUBKEYS} # Add correct key into the list echo ${PUBLIC} >> ${PUBKEYS} From 72d81d7401919c96ad11612a7395442013fe4394 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 20:17:27 +0300 Subject: [PATCH 050/177] tests/local-pull: test "--sign-verify" option Ensure what with this option only signed commit is pulled. Signed-off-by: Denis Pynkin --- tests/test-local-pull.sh | 119 +++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 43 deletions(-) diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index 97bb9954..2b7ca13a 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -28,12 +28,7 @@ unset OSTREE_GPG_HOME skip_without_user_xattrs -if has_gpgme; then - echo "1..8" -else - # Only some tests doesn't need GPG support - echo "1..5" -fi +echo "1..11" setup_test_repository "archive" echo "ok setup" @@ -68,6 +63,49 @@ cmp checkout1.files checkout2.files cmp checkout1.files checkout3.files echo "ok checkouts same" +if has_gpgme; then + # These tests are needed GPG support + mkdir repo4 + ostree_repo_init repo4 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo4 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo + + if ${CMD_PREFIX} ostree --repo=repo4 pull-local --remote=origin --gpg-verify repo test2 2>&1; then + assert_not_reached "GPG verification unexpectedly succeeded" + fi + echo "ok --gpg-verify with no signature" + + ${OSTREE} gpg-sign --gpg-homedir=${TEST_GPG_KEYHOME} test2 ${TEST_GPG_KEYID_1} + + mkdir repo5 + ostree_repo_init repo5 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo5 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo + ${CMD_PREFIX} ostree --repo=repo5 pull-local --remote=origin --gpg-verify repo test2 + echo "ok --gpg-verify" + + mkdir repo6 + ostree_repo_init repo6 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo6 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo + if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then + assert_not_reached "GPG summary verification with no summary unexpectedly succeeded" + fi + + ${OSTREE} summary --update + + if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then + assert_not_reached "GPG summary verification with signed no summary unexpectedly succeeded" + fi + + ${OSTREE} summary --update --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} + + ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1 + + echo "ok --gpg-verify-summary" +else + echo "ok --gpg-verify with no signature | # SKIP due GPG unavailability" + echo "ok --gpg-verify | # SKIP due GPG unavailability" + echo "ok --gpg-verify-summary | # SKIP due GPG unavailability" +fi + mkdir repo7 ostree_repo_init repo7 --mode="archive" ${CMD_PREFIX} ostree --repo=repo7 pull-local repo @@ -78,41 +116,36 @@ for src_object in `find repo/objects -name '*.filez'`; do done echo "ok pull-local z2 to z2 default hardlink" -if ! has_gpgme; then - exit 0 +if has_libsodium; then + gen_ed25519_keys + + mkdir repo8 + ostree_repo_init repo8 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo8 remote add --set=verification-key="${ED25519PUBLIC}" origin repo + cat repo8/config + + if ${CMD_PREFIX} ostree --repo=repo8 pull-local --remote=origin --sign-verify repo test2 2>&1; then + assert_not_reached "Ed25519 signature verification unexpectedly succeeded" + fi + echo "ok --sign-verify with no signature" + + ${OSTREE} sign test2 ${ED25519SECRET} + + mkdir repo9 + ostree_repo_init repo9 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo9 remote add --set=verification-key="$(gen_ed25519_random_public)" origin repo + if ${CMD_PREFIX} ostree --repo=repo9 pull-local --remote=origin --sign-verify repo test2 2>&1; then + assert_not_reached "Ed25519 signature verification unexpectedly succeeded" + fi + echo "ok --sign-verify with wrong signature" + + mkdir repo10 + ostree_repo_init repo10 --mode="archive" + ${CMD_PREFIX} ostree --repo=repo10 remote add --set=verification-key="${ED25519PUBLIC}" origin repo + ${CMD_PREFIX} ostree --repo=repo10 pull-local --remote=origin --sign-verify repo test2 + echo "ok --sign-verify" +else + echo "ok --sign-verify with no signature | # SKIP due libsodium unavailability" + echo "ok --sign-verify with wrong signature | # SKIP due libsodium unavailability" + echo "ok --sign-verify | # SKIP libsodium unavailability" fi - -mkdir repo4 -ostree_repo_init repo4 --mode="archive" -${CMD_PREFIX} ostree --repo=repo4 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo -if ${CMD_PREFIX} ostree --repo=repo4 pull-local --remote=origin --gpg-verify repo test2 2>&1; then - assert_not_reached "GPG verification unexpectedly succeeded" -fi -echo "ok --gpg-verify with no signature" - -${OSTREE} gpg-sign --gpg-homedir=${TEST_GPG_KEYHOME} test2 ${TEST_GPG_KEYID_1} - -mkdir repo5 -ostree_repo_init repo5 --mode="archive" -${CMD_PREFIX} ostree --repo=repo5 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo -${CMD_PREFIX} ostree --repo=repo5 pull-local --remote=origin --gpg-verify repo test2 -echo "ok --gpg-verify" - -mkdir repo6 -ostree_repo_init repo6 --mode="archive" -${CMD_PREFIX} ostree --repo=repo6 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo -if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then - assert_not_reached "GPG summary verification with no summary unexpectedly succeeded" -fi - -${OSTREE} summary --update - -if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then - assert_not_reached "GPG summary verification with signed no summary unexpectedly succeeded" -fi - -${OSTREE} summary --update --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} - -${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1 - -echo "ok --gpg-verify-summary" From 6608436441d390f4b0b9de2bef33503921daa13c Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 20:58:33 +0300 Subject: [PATCH 051/177] bin/remote-add: added "--no-sign-verify" option Option "--no-sign-verify" disable the signature verification while adding remote. Signed-off-by: Denis Pynkin --- src/ostree/ot-remote-builtin-add.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index cea0b274..e4634710 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -28,6 +28,7 @@ static char **opt_set; static gboolean opt_no_gpg_verify; +static gboolean opt_no_sign_verify; static gboolean opt_if_not_exists; static gboolean opt_force; static char *opt_gpg_import; @@ -44,6 +45,7 @@ static char *opt_repo; static GOptionEntry option_entries[] = { { "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, { "no-gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_no_gpg_verify, "Disable GPG verification", NULL }, + { "no-sign-verify", 0, 0, G_OPTION_ARG_NONE, &opt_no_sign_verify, "Disable signature verification", NULL }, { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "force", 0, 0, G_OPTION_ARG_NONE, &opt_force, "Replace the provided remote if it exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, @@ -134,12 +136,18 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio } #ifndef OSTREE_DISABLE_GPGME - if (opt_no_gpg_verify) + /* No signature verification implies no verification for GPG signature as well */ + if (opt_no_gpg_verify || opt_no_sign_verify) g_variant_builder_add (optbuilder, "{s@v}", "gpg-verify", g_variant_new_variant (g_variant_new_boolean (FALSE))); #endif /* OSTREE_DISABLE_GPGME */ + if (opt_no_sign_verify) + g_variant_builder_add (optbuilder, "{s@v}", + "sign-verify", + g_variant_new_variant (g_variant_new_boolean (FALSE))); + if (opt_collection_id != NULL) g_variant_builder_add (optbuilder, "{s@v}", "collection-id", g_variant_new_variant (g_variant_new_take_string (g_steal_pointer (&opt_collection_id)))); From 68aa13550a403eb30e0e0a759198a005fb631445 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sun, 17 Nov 2019 21:01:13 +0300 Subject: [PATCH 052/177] tests: use option "--no-sign-verify" for adding remote Option "--no-sign-verify" disable the signature verification including GPG. So use it in tests instead of "--no-gpg-verification". Signed-off-by: Denis Pynkin --- tests/pull-test.sh | 32 +++++++++++++++---------------- tests/pull-test2.sh | 4 ++-- tests/test-pull-mirrorlist.sh | 6 +++--- tests/test-refs-collections.sh | 4 ++-- tests/test-remote-add.sh | 10 +++++----- tests/test-summary-collections.sh | 4 ++-- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 2cfd8e02..a52adab9 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -29,7 +29,7 @@ function repo_init() { ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@" } -repo_init --no-gpg-verify +repo_init --no-sign-verify # See also the copy of this in basic-test.sh COMMIT_ARGS="" @@ -62,7 +62,7 @@ else fi # Try both syntaxes -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main >out.txt assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched" ${CMD_PREFIX} ostree --repo=repo pull origin:main > out.txt @@ -164,7 +164,7 @@ echo "ok pull (bareuseronly mirror)" # Corruption tests cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify if ! is_bare_user_only_repo repo; then if ! skip_one_without_user_xattrs; then if is_bare_user_only_repo repo; then @@ -216,7 +216,7 @@ if ! skip_one_without_user_xattrs; then done # And ensure the repo is reinitialized - repo_init --no-gpg-verify + repo_init --no-sign-verify echo "ok corruption" fi else @@ -320,7 +320,7 @@ echo "ok pull specific commit" # test pull -T cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main origrev=$(${CMD_PREFIX} ostree --repo=repo rev-parse main) # Check we can pull the same commit with timestamp checking enabled @@ -350,7 +350,7 @@ ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok pull timestamp checking" cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main ${CMD_PREFIX} ostree --repo=repo fsck # Generate a delta from old to current, even though we aren't going to @@ -375,7 +375,7 @@ ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u # Explicitly test delta fetches via ref name as well as commit hash for delta_target in main ${new_rev}; do cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin ${delta_target} >dry-run-pull.txt # Compression can vary, so we support 400-699 @@ -388,7 +388,7 @@ done # Test pull via file:/// - this should still use the deltas path for testing cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo remote delete origin ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} @@ -400,7 +400,7 @@ echo "ok pull file:// + deltas required" # Explicitly test delta fetches via ref name as well as commit hash for delta_target in main ${new_rev}; do cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${delta_target} if test ${delta_target} = main; then @@ -414,12 +414,12 @@ done # Test no-op with deltas: https://github.com/ostreedev/ostree/issues/1321 cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin main ${CMD_PREFIX} ostree --repo=repo fsck @@ -437,7 +437,7 @@ cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt ${CMD_PREFIX} ostree --repo=repo fsck @@ -451,7 +451,7 @@ echo "ok pull byteswapped delta" cd ${test_tmpdir} rm ostree-srv/gnomerepo/deltas -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u -repo_init --no-gpg-verify +repo_init --no-sign-verify if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" fi @@ -459,7 +459,7 @@ assert_file_has_content err.txt "deltas required, but none found" ${CMD_PREFIX} ostree --repo=repo fsck # Now test with a partial commit -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull --commit-metadata-only origin main@${prev_rev} if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" @@ -467,7 +467,7 @@ fi assert_file_has_content err.txt "deltas required, but none found" echo "ok delta required but don't exist" -repo_init --no-gpg-verify +repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin ${new_rev} 2>err.txt; then assert_not_reached "--require-static-deltas unexpectedly succeeded" @@ -595,7 +595,7 @@ if has_gpgme; then fi cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify mv ostree-srv/gnomerepo/refs/heads/main{,.orig} rm ostree-srv/gnomerepo/summary (for x in $(seq 20); do echo "lots of html here "; done) > ostree-srv/gnomerepo/refs/heads/main diff --git a/tests/pull-test2.sh b/tests/pull-test2.sh index 064bbfe6..a0b699ae 100644 --- a/tests/pull-test2.sh +++ b/tests/pull-test2.sh @@ -29,7 +29,7 @@ function repo_init() { ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@" } -repo_init --no-gpg-verify +repo_init --no-sign-verify # See also the copy of this in basic-test.sh COMMIT_ARGS="" @@ -48,7 +48,7 @@ fi echo "1..1" cd ${test_tmpdir} -repo_init --no-gpg-verify +repo_init --no-sign-verify prev_rev=$(ostree --repo=ostree-srv/repo rev-parse ${remote_ref}^) rev=$(ostree --repo=ostree-srv/repo rev-parse ${remote_ref}) ${CMD_PREFIX} ostree --repo=ostree-srv/repo static-delta generate ${remote_ref} diff --git a/tests/test-pull-mirrorlist.sh b/tests/test-pull-mirrorlist.sh index 85ff66e9..ed65eb64 100755 --- a/tests/test-pull-mirrorlist.sh +++ b/tests/test-pull-mirrorlist.sh @@ -75,7 +75,7 @@ EOF cd ${test_tmpdir} mkdir repo ostree_repo_init repo -${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \ +${CMD_PREFIX} ostree --repo=repo remote add origin --no-sign-verify \ mirrorlist=$(cat httpd-address)/ostree/mirrorlist ${CMD_PREFIX} ostree --repo=repo pull origin:main @@ -87,7 +87,7 @@ cd ${test_tmpdir} rm -rf repo mkdir repo ostree_repo_init repo -${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \ +${CMD_PREFIX} ostree --repo=repo remote add origin --no-sign-verify \ --contenturl=mirrorlist=$(cat httpd-address)/ostree/mirrorlist \ $(cat httpd-address)/ostree/gnomerepo ${CMD_PREFIX} ostree --repo=repo pull origin:main @@ -100,7 +100,7 @@ cd ${test_tmpdir} rm -rf repo mkdir repo ostree_repo_init repo -${CMD_PREFIX} ostree --repo=repo remote add origin --no-gpg-verify \ +${CMD_PREFIX} ostree --repo=repo remote add origin --no-sign-verify \ --contenturl=mirrorlist=$(cat httpd-address)/ostree/mirrorlist \ mirrorlist=$(cat httpd-address)/ostree/mirrorlist ${CMD_PREFIX} ostree --repo=repo pull origin:main diff --git a/tests/test-refs-collections.sh b/tests/test-refs-collections.sh index bf233970..d33f498c 100755 --- a/tests/test-refs-collections.sh +++ b/tests/test-refs-collections.sh @@ -112,7 +112,7 @@ mkdir collection-repo ostree_repo_init collection-repo --collection-id org.example.RemoteCollection mkdir -p adir ${CMD_PREFIX} ostree --repo=collection-repo commit --branch=rcommit -m rcommit -s rcommit adir -${CMD_PREFIX} ostree --repo=repo remote add --no-gpg-verify --collection-id org.example.RemoteCollection collection-repo-remote "file://${test_tmpdir}/collection-repo" +${CMD_PREFIX} ostree --repo=repo remote add --no-sign-verify --collection-id org.example.RemoteCollection collection-repo-remote "file://${test_tmpdir}/collection-repo" ${CMD_PREFIX} ostree --repo=repo pull collection-repo-remote rcommit ${CMD_PREFIX} ostree --repo=repo refs --collections > refs @@ -129,7 +129,7 @@ mkdir no-collection-repo ostree_repo_init no-collection-repo mkdir -p adir2 ${CMD_PREFIX} ostree --repo=no-collection-repo commit --branch=rcommit2 -m rcommit2 -s rcommit2 adir2 -${CMD_PREFIX} ostree --repo=repo remote add --no-gpg-verify no-collection-repo-remote "file://${test_tmpdir}/no-collection-repo" +${CMD_PREFIX} ostree --repo=repo remote add --no-sign-verify no-collection-repo-remote "file://${test_tmpdir}/no-collection-repo" ${CMD_PREFIX} ostree --repo=repo pull no-collection-repo-remote rcommit2 ${CMD_PREFIX} ostree --repo=repo refs --collections > refs assert_not_file_has_content refs "rcommit2" diff --git a/tests/test-remote-add.sh b/tests/test-remote-add.sh index bb7eae89..40a32f57 100755 --- a/tests/test-remote-add.sh +++ b/tests/test-remote-add.sh @@ -30,20 +30,20 @@ $OSTREE remote add origin http://example.com/ostree/gnome $OSTREE remote show-url origin >/dev/null echo "ok config" -$OSTREE remote add --no-gpg-verify another http://another.com/repo +$OSTREE remote add --no-sign-verify another http://another.com/repo $OSTREE remote show-url another >/dev/null echo "ok remote no gpg-verify" -if $OSTREE remote add --no-gpg-verify another http://another.example.com/anotherrepo 2>err.txt; then +if $OSTREE remote add --no-sign-verify another http://another.example.com/anotherrepo 2>err.txt; then assert_not_reached "Adding duplicate remote unexpectedly succeeded" fi echo "ok" -$OSTREE remote add --if-not-exists --no-gpg-verify another http://another.example.com/anotherrepo +$OSTREE remote add --if-not-exists --no-sign-verify another http://another.example.com/anotherrepo $OSTREE remote show-url another >/dev/null echo "ok" -$OSTREE remote add --if-not-exists --no-gpg-verify another-noexist http://another-noexist.example.com/anotherrepo +$OSTREE remote add --if-not-exists --no-sign-verify another-noexist http://another-noexist.example.com/anotherrepo $OSTREE remote show-url another-noexist >/dev/null echo "ok" @@ -69,7 +69,7 @@ cd ${test_tmpdir} rm -rf parent-repo ostree_repo_init parent-repo $OSTREE config set core.parent ${test_tmpdir}/parent-repo -${CMD_PREFIX} ostree --repo=parent-repo remote add --no-gpg-verify parent-remote http://parent-remote.example.com/parent-remote +${CMD_PREFIX} ostree --repo=parent-repo remote add --no-sign-verify parent-remote http://parent-remote.example.com/parent-remote $OSTREE remote list > list.txt assert_file_has_content list.txt "origin" assert_file_has_content list.txt "another" diff --git a/tests/test-summary-collections.sh b/tests/test-summary-collections.sh index 777d3d0c..9885c5ea 100755 --- a/tests/test-summary-collections.sh +++ b/tests/test-summary-collections.sh @@ -63,7 +63,7 @@ mkdir collection-repo ostree_repo_init collection-repo --collection-id org.example.RemoteCollection mkdir -p adir ${CMD_PREFIX} ostree --repo=collection-repo commit --branch=rcommit -m rcommit -s rcommit adir -${CMD_PREFIX} ostree --repo=repo remote add --no-gpg-verify --collection-id org.example.RemoteCollection collection-repo-remote "file://${test_tmpdir}/collection-repo" +${CMD_PREFIX} ostree --repo=repo remote add --no-sign-verify --collection-id org.example.RemoteCollection collection-repo-remote "file://${test_tmpdir}/collection-repo" ${CMD_PREFIX} ostree --repo=repo pull collection-repo-remote rcommit ${CMD_PREFIX} ostree --repo=repo summary --update @@ -75,7 +75,7 @@ mkdir no-collection-repo ostree_repo_init no-collection-repo mkdir -p adir2 ${CMD_PREFIX} ostree --repo=no-collection-repo commit --branch=rcommit2 -m rcommit2 -s rcommit2 adir2 -${CMD_PREFIX} ostree --repo=repo remote add --no-gpg-verify no-collection-repo-remote "file://${test_tmpdir}/no-collection-repo" +${CMD_PREFIX} ostree --repo=repo remote add --no-sign-verify no-collection-repo-remote "file://${test_tmpdir}/no-collection-repo" ${CMD_PREFIX} ostree --repo=repo pull no-collection-repo-remote rcommit2 ${CMD_PREFIX} ostree --repo=repo summary --update From fbd2666e076c1cf4bd0a1b13c888b21094b1f97f Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 18 Nov 2019 14:28:40 +0300 Subject: [PATCH 053/177] tests/sign: disable GPG for alternatively signed pull Explicitly disable GPG verification for remote while testing alternative signing mechanism. Signed-off-by: Denis Pynkin --- tests/test-signed-pull.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 238ce8e0..28676b21 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -34,7 +34,7 @@ function repo_init() { rm repo -rf mkdir repo ostree_repo_init repo --mode=${repo_mode} - ${CMD_PREFIX} ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo "$@" + ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo "$@" } function test_signed_pull() { From 485ff7335faf8afb9b4a47eca71cbae10ccd75aa Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 19 Nov 2019 02:44:16 +0300 Subject: [PATCH 054/177] lib/sign: allow to build with glib version less than 2.44 Ubuntu 14.04 uses glib-2.40 which have no some shiny macroses for interface declaration. Signed-off-by: Denis Pynkin --- src/libostree/ostree-autocleanups.h | 1 + src/libostree/ostree-sign-dummy.c | 5 +++++ src/libostree/ostree-sign-dummy.h | 14 ++++++++++++++ src/libostree/ostree-sign-ed25519.c | 5 +++++ src/libostree/ostree-sign-ed25519.h | 15 ++++++++++++++- src/libostree/ostree-sign.c | 2 ++ src/libostree/ostree-sign.h | 18 ++++++++++++++---- src/libostree/ostree.h | 1 + 8 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-autocleanups.h b/src/libostree/ostree-autocleanups.h index c9692ebe..14017012 100644 --- a/src/libostree/ostree-autocleanups.h +++ b/src/libostree/ostree-autocleanups.h @@ -73,6 +73,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderOverride, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) G_DEFINE_AUTO_CLEANUP_FREE_FUNC (OstreeRepoFinderResultv, ostree_repo_finder_result_freev, NULL) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSign, g_object_unref) #endif G_END_DECLS diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index e2d1fe56..48190149 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -24,6 +24,7 @@ #include "config.h" +#include #include "ostree-sign-dummy.h" #include @@ -43,6 +44,10 @@ struct _OstreeSignDummy gchar *signature_ascii; }; +#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSignDummy, g_object_unref) +#endif + static void ostree_sign_dummy_iface_init (OstreeSignInterface *self); diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index a0d10e1d..f80f8682 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -32,12 +32,26 @@ G_BEGIN_DECLS #define OSTREE_TYPE_SIGN_DUMMY (ostree_sign_dummy_get_type ()) +_OSTREE_PUBLIC +GType ostree_sign_dummy_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeSignDummy OstreeSignDummy; +typedef struct { GObjectClass parent_class; } OstreeSignDummyClass; + +static inline OstreeSignDummy *OSTREE_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_sign_dummy_get_type (), OstreeSignDummy); } +static inline gboolean OSTREE_IS_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_sign_dummy_get_type ()); } + +G_GNUC_END_IGNORE_DEPRECATIONS + +/* Have to use glib-2.44 for this _OSTREE_PUBLIC G_DECLARE_FINAL_TYPE (OstreeSignDummy, ostree_sign_dummy, OSTREE, SIGN_DUMMY, GObject) +*/ const gchar * ostree_sign_dummy_get_name (OstreeSign *self); diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 0c7cd951..2d5bdb16 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -25,6 +25,7 @@ #include "config.h" +#include #include "ostree-sign-ed25519.h" #ifdef HAVE_LIBSODIUM #include @@ -47,6 +48,10 @@ struct _OstreeSignEd25519 GList *revoked_keys; }; +#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSignEd25519, g_object_unref) +#endif + static void ostree_sign_ed25519_iface_init (OstreeSignInterface *self); diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index bced1cdf..6e5dd665 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -32,13 +32,26 @@ G_BEGIN_DECLS #define OSTREE_TYPE_SIGN_ED25519 (ostree_sign_ed25519_get_type ()) +_OSTREE_PUBLIC +GType ostree_sign_ed25519_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeSignEd25519 OstreeSignEd25519; +typedef struct { GObjectClass parent_class; } OstreeSignEd25519Class; + +static inline OstreeSignEd25519 *OSTREE_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_sign_ed25519_get_type (), OstreeSignEd25519); } +static inline gboolean OSTREE_IS_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_sign_ed25519_get_type ()); } + +G_GNUC_END_IGNORE_DEPRECATIONS + +/* Have to use glib-2.44 for this _OSTREE_PUBLIC G_DECLARE_FINAL_TYPE (OstreeSignEd25519, ostree_sign_ed25519, OSTREE, SIGN_ED25519, GObject) - +*/ gboolean ostree_sign_ed25519_data (OstreeSign *self, GBytes *data, diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 95319f67..75db0b26 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -47,6 +47,8 @@ #include "ostree-sign-ed25519.h" #endif +#include "ostree-autocleanups.h" + #undef G_LOG_DOMAIN #define G_LOG_DOMAIN "OSTreeSign" diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 87ed25ce..1415becb 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -33,16 +33,26 @@ #include "ostree-remote.h" #include "ostree-types.h" -/* Special remote */ -#define OSTREE_SIGN_ALL_REMOTES "__OSTREE_ALL_REMOTES__" - - G_BEGIN_DECLS #define OSTREE_TYPE_SIGN (ostree_sign_get_type ()) +_OSTREE_PUBLIC +GType ostree_sign_get_type (void); + +G_GNUC_BEGIN_IGNORE_DEPRECATIONS +typedef struct _OstreeSign OstreeSign; +typedef struct _OstreeSignInterface OstreeSignInterface; + +static inline OstreeSign *OSTREE_SIGN (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_sign_get_type (), OstreeSign); } +static inline gboolean OSTREE_IS_SIGN (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_sign_get_type ()); } +static inline OstreeSignInterface *OSTREE_SIGN_GET_IFACE (gpointer ptr) { return G_TYPE_INSTANCE_GET_INTERFACE (ptr, ostree_sign_get_type (), OstreeSignInterface); } +G_GNUC_END_IGNORE_DEPRECATIONS + +/* Have to use glib-2.44 for this _OSTREE_PUBLIC G_DECLARE_INTERFACE (OstreeSign, ostree_sign, OSTREE, SIGN, GObject) +*/ struct _OstreeSignInterface { diff --git a/src/libostree/ostree.h b/src/libostree/ostree.h index 49ca919c..0308d0ed 100644 --- a/src/libostree/ostree.h +++ b/src/libostree/ostree.h @@ -40,5 +40,6 @@ #include #include #include +#include #include #include From 3063a0a838981aa0e04e6f2acb2cc78ce6bb068d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 25 Nov 2019 22:20:03 +0300 Subject: [PATCH 055/177] lib/sign: use separate public and secret keys for 'dummy' The initial implementation with single key for secret and public parts doesn't allow to test pulling with several signing engines used. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-dummy.c | 39 +++++++++++++++++++++---------- src/libostree/ostree-sign-dummy.h | 4 +++- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 48190149..034c7d6b 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -36,12 +36,11 @@ #define OSTREE_SIGN_METADATA_DUMMY_KEY "ostree.sign.dummy" #define OSTREE_SIGN_METADATA_DUMMY_TYPE "aay" -#define OSTREE_SIGN_DUMMY_SIGNATURE "dummysign" - struct _OstreeSignDummy { GObject parent; - gchar *signature_ascii; + gchar *sk_ascii; + gchar *pk_ascii; }; #ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC @@ -64,8 +63,10 @@ ostree_sign_dummy_iface_init (OstreeSignInterface *self) self->data_verify = ostree_sign_dummy_data_verify; self->metadata_key = ostree_sign_dummy_metadata_key; self->metadata_format = ostree_sign_dummy_metadata_format; - self->set_sk = ostree_sign_dummy_set_key; - self->set_pk = ostree_sign_dummy_set_key; + self->set_sk = ostree_sign_dummy_set_sk; + self->set_pk = ostree_sign_dummy_set_pk; + /* Implementation for dummy engine just load the single public key */ + self->add_pk = ostree_sign_dummy_set_pk; } static void @@ -79,19 +80,32 @@ ostree_sign_dummy_init (OstreeSignDummy *self) { g_debug ("%s enter", __FUNCTION__); - self->signature_ascii = g_strdup(OSTREE_SIGN_DUMMY_SIGNATURE); + self->sk_ascii = NULL; + self->pk_ascii = NULL; } -gboolean ostree_sign_dummy_set_key (OstreeSign *self, GVariant *key, GError **error) +gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **error) { g_debug ("%s enter", __FUNCTION__); OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); - if (sign->signature_ascii != NULL) - g_free(sign->signature_ascii); + g_free(sign->sk_ascii); - sign->signature_ascii = g_variant_dup_string (key, 0); + sign->sk_ascii = g_variant_dup_string (key, 0); + + return TRUE; +} + +gboolean ostree_sign_dummy_set_pk (OstreeSign *self, GVariant *key, GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + + OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + + g_free(sign->pk_ascii); + + sign->pk_ascii = g_variant_dup_string (key, 0); return TRUE; } @@ -108,7 +122,7 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); - *signature = g_bytes_new (sign->signature_ascii, strlen(sign->signature_ascii)); + *signature = g_bytes_new (sign->sk_ascii, strlen(sign->sk_ascii)); return TRUE; } @@ -174,8 +188,9 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, g_bytes_get_data (signature, &sign_size); g_autofree gchar *sign_ascii = g_strndup(g_bytes_get_data (signature, NULL), sign_size); g_debug("Read signature %d: %s", (gint)i, sign_ascii); + g_debug("Stored signature %d: %s", (gint)i, sign->pk_ascii); - if (!g_strcmp0(sign_ascii, sign->signature_ascii)) + if (!g_strcmp0(sign_ascii, sign->pk_ascii)) ret = TRUE; } if (ret == FALSE && *error == NULL) diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index f80f8682..6fc3a363 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -69,7 +69,9 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self); -gboolean ostree_sign_dummy_set_key (OstreeSign *self, GVariant *key, GError **error); +gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **error); +gboolean ostree_sign_dummy_set_pk (OstreeSign *self, GVariant *key, GError **error); +gboolean ostree_sign_dummy_add_pk (OstreeSign *self, GVariant *key, GError **error); G_END_DECLS From 5dca74fab7a5419e4151de4b13de8a351b8b724b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 25 Nov 2019 19:50:07 +0000 Subject: [PATCH 056/177] tests/sign: add verification key for pulling with dummy After splitting out the common key to secret/public inside the dummy engine we need to pass the the public key for remote with dummy engine usage. Signed-off-by: Denis Pynkin --- tests/test-signed-pull.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 28676b21..e57a40f3 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -34,7 +34,7 @@ function repo_init() { rm repo -rf mkdir repo ostree_repo_init repo --mode=${repo_mode} - ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo "$@" + ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false --set=sign-verify-summary=false origin $(cat httpd-address)/ostree/gnomerepo "$@" } function test_signed_pull() { @@ -66,6 +66,7 @@ function test_signed_pull() { DUMMYSIGN="dummysign" COMMIT_ARGS="--sign=${DUMMYSIGN} --sign-type=dummy" repo_init --set=sign-verify=true +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${DUMMYSIGN}" test_signed_pull "dummy" From 5cd822ae0528ac3b573e7d01c1a01f47f4722396 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 25 Nov 2019 22:53:28 +0300 Subject: [PATCH 057/177] lib/sign: fix the false failure while loading keys Usage of 'g_warning()' inside keys loading funcrion lead to false failure: the key loading attempt for the wrong engine breaks the pulling process instead of trying to use this key with correct engine. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4dd0f11a..78bb32f3 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1492,7 +1492,6 @@ _load_public_keys (OtPullData *pull_data, gboolean loaded_inlined = TRUE; g_autoptr (GError) error = NULL; - /* Load keys for remote from file */ ostree_repo_get_remote_option (pull_data->repo, pull_data->remote_name, "verification-file", NULL, @@ -1531,12 +1530,9 @@ _load_public_keys (OtPullData *pull_data, loaded_from_file = TRUE; else { - if (error == NULL) - g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, - "unknown reason"); - - g_warning("Unable to load public keys from file '%s': %s", - pk_file, error->message); + g_assert (error); + g_debug("Unable to load public keys for '%s' from file '%s': %s", + ostree_sign_get_name(sign), pk_file, error->message); g_clear_error (&error); } } @@ -1557,8 +1553,8 @@ _load_public_keys (OtPullData *pull_data, g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, "unknown reason"); - g_warning("Unable to load public key '%s': %s", - pk_ascii, error->message); + g_debug("Unable to load public key '%s' for '%s': %s", + pk_ascii, ostree_sign_get_name(sign), error->message); g_clear_error (&error); } } From acace9b95ab63b64f7d0bc349d94372ccd1a9575 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 09:40:57 +0000 Subject: [PATCH 058/177] tests/sign: allow to start pull test without libsodium Allow to run the pulling test if there is no ed25519 support. Test the signed pull only with dummy engine. Fixed tests names. Signed-off-by: Denis Pynkin --- tests/test-signed-pull.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index e57a40f3..4d8d7aab 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..7" +echo "1..6" setup_fake_remote_repo1 "archive" @@ -39,6 +39,7 @@ function repo_init() { function test_signed_pull() { local sign_type="$1" + local comment="$2" cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} \ -b main -s "A signed commit" --tree=ref=main @@ -56,19 +57,26 @@ function test_signed_pull() { # ok now check that we can pull correctly mv $remotesig.bak $remotesig ${CMD_PREFIX} ostree --repo=repo pull origin main - echo "ok pull ${sign_type} signed commit" + echo "ok ${sign_type}${comment} pull signed commit" rm $localsig ${CMD_PREFIX} ostree --repo=repo pull origin main test -f $localsig - echo "ok re-pull ${sign_type} signature for stored commit" + echo "ok ${sign_type}${comment} re-pull signature for stored commit" } DUMMYSIGN="dummysign" COMMIT_ARGS="--sign=${DUMMYSIGN} --sign-type=dummy" repo_init --set=sign-verify=true ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${DUMMYSIGN}" -test_signed_pull "dummy" +test_signed_pull "dummy" "" +if ! has_libsodium; then + echo "ok ed25519-key pull signed commit # SKIP due libsodium unavailability" + echo "ok ed25519-key re-pull signature for stored commit # SKIP due libsodium unavailability" + echo "ok ed25519-file pull signed commit # SKIP due libsodium unavailability" + echo "ok ed25519-file re-pull signature for stored commit # SKIP due libsodium unavailability" + exit 0 +fi # Test ostree sign with 'ed25519' module gen_ed25519_keys @@ -80,7 +88,7 @@ COMMIT_ARGS="--sign=${SECRET} --sign-type=ed25519" repo_init --set=sign-verify=true ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${PUBLIC}" -test_signed_pull "ed25519" +test_signed_pull "ed25519" "key" # Prepare files with public ed25519 signatures PUBKEYS="$(mktemp -p ${test_tmpdir} ed25519_XXXXXX.ed25519)" @@ -95,6 +103,5 @@ echo ${PUBLIC} >> ${PUBKEYS} repo_init --set=sign-verify=true ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "${PUBKEYS}" -test_signed_pull "ed25519" +test_signed_pull "ed25519" "file" -echo "ok verify ed25519 keys file" From 0bdcf14d567d6cd0abaa8e18d60a165c9f011aec Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 12:44:44 +0300 Subject: [PATCH 059/177] lib/sign: new function for summary file signing Add function `ostree_sign_summary()` allowing to sign the summary file. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree-devel.sym | 1 + src/libostree/ostree-sign.c | 96 +++++++++++++++++++++++++++++++ src/libostree/ostree-sign.h | 6 ++ 4 files changed, 104 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 9b71d610..a5a632a5 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -723,6 +723,7 @@ ostree_sign_clear_keys ostree_sign_load_pk ostree_sign_set_pk ostree_sign_set_sk +ostree_sign_summary ostree_sign_get_type ostree_sign_dummy_get_type diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index a10ec266..29221f5d 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -35,6 +35,7 @@ global: ostree_sign_set_pk; ostree_sign_add_pk; ostree_sign_set_sk; + ostree_sign_summary; ostree_sign_dummy_get_type; ostree_sign_ed25519_get_type; } LIBOSTREE_2020.1; diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 75db0b26..e7962425 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -48,6 +48,7 @@ #endif #include "ostree-autocleanups.h" +#include "ostree-repo-private.h" #undef G_LOG_DOMAIN #define G_LOG_DOMAIN "OSTreeSign" @@ -594,3 +595,98 @@ ostree_sign_get_by_name (const gchar *name, GError **error) return sign; } + +/** + * ostree_sign_summary: + * @self: Self + * @repo: ostree repository + * @keys: keys -- GVariant containing keys as GVarints specific to signature type. + * @cancellable: A #GCancellable + * @error: a #GError + * + * Add a signature to a summary file. + * Based on ostree_repo_add_gpg_signature_summary implementation. + * + * Returns: @TRUE if summary file has been signed with all provided keys + */ +gboolean +ostree_sign_summary (OstreeSign *self, + OstreeRepo *repo, + GVariant *keys, + GCancellable *cancellable, + GError **error) +{ + g_debug ("%s enter", __FUNCTION__); + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); + + gboolean ret = FALSE; + + g_autoptr(GVariant) normalized = NULL; + g_autoptr(GBytes) summary_data = NULL; + g_autoptr(GVariant) metadata = NULL; + + glnx_autofd int fd = -1; + if (!glnx_openat_rdonly (repo->repo_dir_fd, "summary", TRUE, &fd, error)) + goto out; + summary_data = ot_fd_readall_or_mmap (fd, 0, error); + if (!summary_data) + goto out; + + /* Note that fd is reused below */ + glnx_close_fd (&fd); + + if (!ot_openat_ignore_enoent (repo->repo_dir_fd, "summary.sig", &fd, error)) + goto out; + if (fd >= 0) + { + if (!ot_variant_read_fd (fd, 0, OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + FALSE, &metadata, error)) + goto out; + } + + if (g_variant_n_children(keys) == 0) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No keys passed for signing summary"); + goto out; + } + + GVariantIter *iter; + GVariant *key; + + g_variant_get (keys, "av", &iter); + while (g_variant_iter_loop (iter, "v", &key)) + { + g_autoptr (GBytes) signature = NULL; + + if (!ostree_sign_set_sk (self, key, error)) + goto out; + + if (!ostree_sign_data (self, + summary_data, + &signature, + cancellable, + error)) + goto out; + + g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata); + metadata = + _sign_detached_metadata_append (self, old_metadata, signature); + } + g_variant_iter_free (iter); + + normalized = g_variant_get_normal_form (metadata); + if (!_ostree_repo_file_replace_contents (repo, + repo->repo_dir_fd, + "summary.sig", + g_variant_get_data (normalized), + g_variant_get_size (normalized), + cancellable, error)) + goto out; + + ret = TRUE; + +out: + return ret; +} diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 1415becb..678f182d 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -152,5 +152,11 @@ GStrv ostree_sign_list_names(void); _OSTREE_PUBLIC OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error); +_OSTREE_PUBLIC +gboolean ostree_sign_summary (OstreeSign *self, + OstreeRepo *repo, + GVariant *keys, + GCancellable *cancellable, + GError **error); G_END_DECLS From 137306f6f347788205d0b4756ac09709139a7b0c Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 12:48:35 +0300 Subject: [PATCH 060/177] bin/summary: add signing with alternative mechanism Allow to sign the summary file with alternative signing mechanism. Added new options: - --sign-type -- select the engine (defaults to ed25519) - --sign -- secret key to use for signing Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-summary.c | 80 ++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index 0f70f071..de6df835 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -27,10 +27,13 @@ #include "ot-builtins.h" #include "ostree.h" #include "otutil.h" +#include "ostree-sign.h" static gboolean opt_update, opt_view, opt_raw; -static char **opt_key_ids; +static char **opt_gpg_key_ids; static char *opt_gpg_homedir; +static char **opt_key_ids; +static char *opt_sign_name; static char **opt_metadata; /* ATTENTION: @@ -42,8 +45,10 @@ static GOptionEntry options[] = { { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, { "view", 'v', 0, G_OPTION_ARG_NONE, &opt_view, "View the local summary file", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "View the raw bytes of the summary file", NULL }, - { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, + { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_gpg_key_ids, "GPG Key ID to sign the summary with", "KEY-ID"}, { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"}, + { "sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "Key ID to sign the summary with", "KEY-ID"}, + { "sign-type", 0, 0, G_OPTION_ARG_STRING, &opt_sign_name, "Signature type to use (defaults to 'ed25519')", "NAME"}, { "add-metadata", 'm', 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata, "Additional metadata field to add to the summary", "KEY=VALUE" }, { NULL } }; @@ -87,6 +92,7 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati { g_autoptr(GOptionContext) context = NULL; g_autoptr(OstreeRepo) repo = NULL; + g_autoptr (OstreeSign) sign = NULL; OstreeDumpFlags flags = OSTREE_DUMP_NONE; context = g_option_context_new (""); @@ -94,6 +100,17 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati if (!ostree_option_context_parse (context, options, &argc, &argv, invocation, &repo, cancellable, error)) return FALSE; + /* Initialize crypto system */ + if (opt_key_ids) + { + if (!opt_sign_name) + opt_sign_name = "ed25519"; + + sign = ostree_sign_get_by_name (opt_sign_name, error); + if (sign == NULL) + return FALSE; + } + if (opt_update) { g_autoptr(GVariant) additional_metadata = NULL; @@ -164,10 +181,9 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati new_summary_commit, repo_file, &new_ostree_metadata_checksum, NULL, error)) return FALSE; - - if (opt_key_ids != NULL) + if (opt_gpg_key_ids != NULL) { - for (const char * const *iter = (const char * const *) opt_key_ids; + for (const char * const *iter = (const char * const *) opt_gpg_key_ids; iter != NULL && *iter != NULL; iter++) { const char *key_id = *iter; @@ -182,6 +198,27 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati } } + if (opt_key_ids) + { + char **iter; + for (iter = opt_key_ids; iter && *iter; iter++) + { + const char *keyid = *iter; + g_autoptr (GVariant) secret_key = NULL; + + secret_key = g_variant_new_string (keyid); + if (!ostree_sign_set_sk (sign, secret_key, error)) + return FALSE; + + if (!ostree_sign_commit (sign, + repo, + new_ostree_metadata_checksum, + cancellable, + error)) + return FALSE; + } + } + ostree_repo_transaction_set_collection_ref (repo, &collection_ref, new_ostree_metadata_checksum); @@ -194,16 +231,45 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati return FALSE; #ifndef OSTREE_DISABLE_GPGME - if (opt_key_ids) + if (opt_gpg_key_ids) { if (!ostree_repo_add_gpg_signature_summary (repo, - (const gchar **) opt_key_ids, + (const gchar **) opt_gpg_key_ids, opt_gpg_homedir, cancellable, error)) return FALSE; } #endif + if (opt_key_ids) + { + g_autoptr (GVariant) secret_keys = NULL; + g_autoptr (GVariantBuilder) sk_builder = NULL; + + sk_builder = g_variant_builder_new (G_VARIANT_TYPE_ARRAY); + + char **iter; + for (iter = opt_key_ids; iter && *iter; iter++) + { + const char *keyid = *iter; + GVariant *secret_key = NULL; + + /* Currently only strings are used as keys + * for supported signature types */ + secret_key = g_variant_new_string (keyid); + + g_variant_builder_add (sk_builder, "v", secret_key); + } + + secret_keys = g_variant_builder_end (sk_builder); + + if (! ostree_sign_summary (sign, + repo, + secret_keys, + cancellable, + error)) + return FALSE; + } } else if (opt_view || opt_raw) { From 40b80344f87b1ddcbf0d8190fbaf80df956172c0 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 09:57:35 +0000 Subject: [PATCH 061/177] lib/repo-pull: verify signature on summary pull Add signature verification on summary file pulling. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 180 ++++++++++++++++++++++--------- 1 file changed, 128 insertions(+), 52 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 78bb32f3..c6a94d72 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -108,6 +108,7 @@ typedef struct { gboolean gpg_verify; gboolean gpg_verify_summary; gboolean sign_verify; + gboolean sign_verify_summary; gboolean require_static_deltas; gboolean disable_static_deltas; gboolean has_tombstone_commits; @@ -1563,6 +1564,51 @@ _load_public_keys (OtPullData *pull_data, return (loaded_from_file || loaded_inlined); } +static gboolean +_ostree_repo_sign_verify (OtPullData *pull_data, + GBytes *signed_data, + GVariant *metadata) +{ + /* list all signature types in detached metadata and check if signed by any? */ + g_auto (GStrv) names = ostree_sign_list_names(); + for (char **iter=names; iter && *iter; iter++) + { + g_autoptr (OstreeSign) sign = NULL; + g_autoptr (GVariant) signatures = NULL; + const gchar *signature_key = NULL; + GVariantType *signature_format = NULL; + g_autoptr (GError) local_error = NULL; + + if ((sign = ostree_sign_get_by_name (*iter, &local_error)) == NULL) + continue; + + signature_key = ostree_sign_metadata_key (sign); + signature_format = (GVariantType *) ostree_sign_metadata_format (sign); + + signatures = g_variant_lookup_value (metadata, + signature_key, + signature_format); + + /* If not found signatures for requested signature subsystem */ + if (!signatures) + continue; + + /* Try to load public key(s) according remote's configuration */ + if (!_load_public_keys (pull_data, sign)) + continue; + + /* Return true if any signature fit to pre-loaded public keys. + * If no keys configured -- then system configuration will be used */ + if (ostree_sign_data_verify (sign, + signed_data, + signatures, + &local_error)) + return TRUE; + } + + return FALSE; +} + static gboolean ostree_verify_unwritten_commit (OtPullData *pull_data, const char *checksum, @@ -1572,21 +1618,24 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, GCancellable *cancellable, GError **error) { + + if (pull_data->gpg_verify || pull_data->sign_verify) + /* Shouldn't happen, but see comment in process_verify_result() */ + if (g_hash_table_contains (pull_data->verified_commits, checksum)) + return TRUE; + + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); + #ifndef OSTREE_DISABLE_GPGME if (pull_data->gpg_verify) { const char *keyring_remote = NULL; - /* Shouldn't happen, but see comment in process_verify_result() */ - if (g_hash_table_contains (pull_data->verified_commits, checksum)) - return TRUE; - if (ref != NULL) keyring_remote = g_hash_table_lookup (pull_data->ref_keyring_map, ref); if (keyring_remote == NULL) keyring_remote = pull_data->remote_name; - g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); g_autoptr(OstreeGpgVerifyResult) result = _ostree_repo_gpg_verify_with_metadata (pull_data->repo, signed_data, detached_metadata, @@ -1606,59 +1655,17 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, "Can't verify commit without detached metadata"); return FALSE; } - /* Shouldn't happen, but see comment in process_verify_result() */ - if (g_hash_table_contains (pull_data->verified_commits, checksum)) - return TRUE; - gboolean ret = FALSE; - g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); - - /* list all signature types in detached metadata and check if signed by any? */ - g_auto (GStrv) names = ostree_sign_list_names(); - for (guint i=0; i < g_strv_length (names); i++) + if (!_ostree_repo_sign_verify (pull_data, signed_data, detached_metadata)) { - g_autoptr (OstreeSign) sign = NULL; - g_autoptr (GVariant) signatures = NULL; - const gchar *signature_key = NULL; - GVariantType *signature_format = NULL; - g_autoptr (GError) local_error = NULL; - - if ((sign = ostree_sign_get_by_name (names[i], &local_error)) == NULL) - continue; - - signature_key = ostree_sign_metadata_key (sign); - signature_format = (GVariantType *) ostree_sign_metadata_format (sign); - - signatures = g_variant_lookup_value (detached_metadata, - signature_key, - signature_format); - - /* If not found signatures for requested signature subsystem */ - if (!signatures) - continue; - - /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (pull_data, sign)) - continue; - - /* Return true if any signature fit to pre-loaded public keys. - * If no keys configured -- then system configuration will be used */ - if (ostree_sign_data_verify (sign, - signed_data, - signatures, - &local_error)) - ret = TRUE; + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't verify commit"); + return FALSE; } /* Mark the commit as verified to avoid double verification * see process_verify_result () for rationale */ - if (ret) - g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); - else - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't verify commit"); - - return ret; + g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); } return TRUE; @@ -3773,6 +3780,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; gboolean opt_sign_verify_set = FALSE; + gboolean opt_sign_verify_summary_set = FALSE; gboolean opt_collection_refs_set = FALSE; gboolean opt_n_network_retries_set = FALSE; gboolean opt_ref_keyring_map_set = FALSE; @@ -3809,6 +3817,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary); opt_sign_verify_set = g_variant_lookup (options, "sign-verify", "b", &pull_data->sign_verify); + opt_sign_verify_summary_set = + g_variant_lookup (options, "sign-verify-summary", "b", &pull_data->sign_verify_summary); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &pull_data->disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas); @@ -3996,6 +4006,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, "sign-verify", FALSE, &pull_data->sign_verify, error)) goto out; + if (!opt_sign_verify_summary_set) + if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, + "sign-verify-summary", FALSE, + &pull_data->sign_verify_summary, error)) + goto out; /* NOTE: If changing this, see the matching implementation in * ostree-sysroot-upgrader.c @@ -4377,6 +4392,67 @@ ostree_repo_pull_with_options (OstreeRepo *self, } #endif /* OSTREE_DISABLE_GPGME */ + if (pull_data->sign_verify_summary) + { + if (!bytes_sig && pull_data->sign_verify_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); + goto out; + } + if (bytes_summary && bytes_sig) + { + g_autoptr(GVariant) signatures = NULL; + + signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + bytes_sig, FALSE); + + + if (!_ostree_repo_sign_verify (pull_data, bytes_summary, signatures)) + { + gboolean ret = FALSE; + + if (summary_from_cache) + { + /* The cached summary doesn't match, fetch a new one and verify again */ + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote %s cached summary invalid and " + "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", + pull_data->remote_name); + goto out; + } + else + g_debug ("Remote %s cached summary invalid, pulling new version", + pull_data->remote_name); + + summary_from_cache = FALSE; + g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + pull_data->n_network_retries, + &bytes_summary, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) + goto out; + + if (_ostree_repo_sign_verify (pull_data, bytes_summary, signatures)) + ret = TRUE; + } + + if (!ret) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't verify summary"); + goto out; + } + } + } + } + if (bytes_summary) { pull_data->summary_data = g_bytes_ref (bytes_summary); From 1de2efa2ed95cbcabdd58f15c974e3fd55c30a70 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 13:01:36 +0300 Subject: [PATCH 062/177] tests/sign: new test for summary file verification Add test for signature verification of summary file during the pull. Adopted version of GPG tests from `test-pull-summary-sigs.sh`. Signed-off-by: Denis Pynkin --- Makefile-tests.am | 6 +- tests/test-signed-pull-summary.sh | 287 ++++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+), 5 deletions(-) create mode 100755 tests/test-signed-pull-summary.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 505245cd..3270bd9c 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -138,13 +138,9 @@ _installed_or_uninstalled_test_scripts = \ tests/test-pull-collections.sh \ tests/test-config.sh \ tests/test-signed-commit.sh \ - $(NULL) - -if USE_LIBSODIUM -_installed_or_uninstalled_test_scripts += \ tests/test-signed-pull.sh \ + tests/test-signed-pull-summary.sh \ $(NULL) -endif if USE_GPGME _installed_or_uninstalled_test_scripts += \ diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh new file mode 100755 index 00000000..7b18cf65 --- /dev/null +++ b/tests/test-signed-pull-summary.sh @@ -0,0 +1,287 @@ +#!/bin/bash +# +# Copyright (C) 2019 Collabora Ltd. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# Based on test-pull-summary-sigs.sh test. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo "1..14" + +repo_reinit () { + ARGS="$*" + cd ${test_tmpdir} + rm -rf repo + mkdir repo + ostree_repo_init repo --mode=archive + ${OSTREE} --repo=repo remote add \ + --set=gpg-verify=false --set=gpg-verify-summary=false \ + --set=sign-verify=false --set=sign-verify-summary=true \ + ${ARGS} origin $(cat httpd-address)/ostree/gnomerepo +} + +for engine in dummy ed25519 +do + case "${engine}" in + dummy) + # Tests with dummy engine + SIGN_KEY="dummysign" + PUBLIC_KEY="dummysign" + ;; + ed25519) + if ! has_libsodium; then + echo "ok ${engine} pull mirror summary # SKIP due libsodium unavailability" + echo "ok ${engine} pull with signed summary # SKIP due libsodium unavailability" + echo "ok ${engine} prune summary cache # SKIP due libsodium unavailability" + echo "ok ${engine} pull with signed summary and cachedir # SKIP due libsodium unavailability" + echo "ok ${engine} pull with invalid ${engine} summary signature fails # SKIP due libsodium unavailability" + echo "ok ${engine} pull delta with signed summary # SKIP due libsodium unavailability" + + continue + fi + gen_ed25519_keys + SIGN_KEY="${ED25519SECRET}" + PUBLIC_KEY="${ED25519PUBLIC}" + ;; + *) + fatal "Unsupported engine ${engine}" + ;; + esac + + COMMIT_SIGN="--sign-type=${engine} --sign=${SIGN_KEY}" + + # clenup the testdir prior the next engine + rm -rf ${test_tmpdir}/ostree-srv ${test_tmpdir}/gnomerepo ${test_tmpdir}/httpd ${test_tmpdir}/repo ${test_tmpdir}/cachedir\ + ${test_tmpdir}/main-copy ${test_tmpdir}/other-copy ${test_tmpdir}/yet-another-copy + + setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" + + # Now, setup multiple branches + mkdir ${test_tmpdir}/ostree-srv/other-files + cd ${test_tmpdir}/ostree-srv/other-files + echo 'hello world another object' > hello-world + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_SIGN} -b other -s "A commit" -m "Another Commit body" + + mkdir ${test_tmpdir}/ostree-srv/yet-other-files + cd ${test_tmpdir}/ostree-srv/yet-other-files + echo 'hello world yet another object' > yet-another-hello-world + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_SIGN} -b yet-another -s "A commit" -m "Another Commit body" + + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u + + prev_dir=`pwd` + cd ${test_tmpdir} + ostree_repo_init repo --mode=archive + ${CMD_PREFIX} ostree --repo=repo remote add \ + --set=gpg-verify=false --set=gpg-verify-summary=false \ + --set=sign-verify=false --set=sign-verify-summary=false \ + origin $(cat httpd-address)/ostree/gnomerepo + ${CMD_PREFIX} ostree --repo=repo pull --mirror origin + assert_has_file repo/summary + ${CMD_PREFIX} ostree --repo=repo checkout -U main main-copy + assert_file_has_content main-copy/baz/cow "moo" + ${CMD_PREFIX} ostree --repo=repo checkout -U other other-copy + assert_file_has_content other-copy/hello-world "hello world another object" + ${CMD_PREFIX} ostree --repo=repo checkout -U yet-another yet-another-copy + assert_file_has_content yet-another-copy/yet-another-hello-world "hello world yet another object" + ${CMD_PREFIX} ostree --repo=repo fsck + echo "ok ${engine} pull mirror summary" + + + cd $prev_dir + + ${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} + + cd ${test_tmpdir} + repo_reinit --set=verification-key=${PUBLIC_KEY} + ${OSTREE} --repo=repo pull origin main + assert_has_file repo/tmp/cache/summaries/origin + assert_has_file repo/tmp/cache/summaries/origin.sig + + rm repo/tmp/cache/summaries/origin + ${OSTREE} --repo=repo pull origin main + assert_has_file repo/tmp/cache/summaries/origin + + echo "ok ${engine} pull with signed summary" + + touch repo/tmp/cache/summaries/foo + touch repo/tmp/cache/summaries/foo.sig + ${OSTREE} --repo=repo prune + assert_not_has_file repo/tmp/cache/summaries/foo + assert_not_has_file repo/tmp/cache/summaries/foo.sig + assert_has_file repo/tmp/cache/summaries/origin + assert_has_file repo/tmp/cache/summaries/origin.sig + echo "ok ${engine} prune summary cache" + + cd ${test_tmpdir} + repo_reinit --set=verification-key=${PUBLIC_KEY} + mkdir cachedir + ${OSTREE} --repo=repo pull --cache-dir=cachedir origin main + assert_not_has_file repo/tmp/cache/summaries/origin + assert_not_has_file repo/tmp/cache/summaries/origin.sig + assert_has_file cachedir/summaries/origin + assert_has_file cachedir/summaries/origin.sig + + rm cachedir/summaries/origin + ${OSTREE} --repo=repo pull --cache-dir=cachedir origin main + assert_not_has_file repo/tmp/cache/summaries/origin + assert_has_file cachedir/summaries/origin + + echo "ok ${engine} pull with signed summary and cachedir" + + cd ${test_tmpdir} + repo_reinit --set=verification-key=${PUBLIC_KEY} + mv ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{,.good} + echo invalid > ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig + if ${OSTREE} --repo=repo pull origin main 2>err.txt; then + assert_not_reached "Successful pull with invalid ${engine} signature" + fi + assert_file_has_content err.txt "Can't verify summary" + mv ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.good,} + echo "ok ${engine} pull with invalid ${engine} summary signature fails" + + # Generate a delta + ${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo static-delta generate --empty main + ${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} + + cd ${test_tmpdir} + repo_reinit --set=verification-key=${PUBLIC_KEY} + ${OSTREE} --repo=repo pull origin main + echo "ok ${engine} pull delta with signed summary" + +done + +if ! has_libsodium; then + echo "ok ${engine} pull with signed summary remote old summary # SKIP due libsodium unavailability" + echo "ok ${engine} pull with signed summary broken cache # SKIP due libsodium unavailability" + exit 0 +fi + +gen_ed25519_keys +SIGN_KEY="${ED25519SECRET}" +PUBLIC_KEY="${ED25519PUBLIC}" +COMMIT_SIGN="--sign-type=${engine} --sign=${SIGN_KEY}" + + +# Verify 'ostree remote summary' output. +${OSTREE} --repo=repo remote summary origin > summary.txt +assert_file_has_content summary.txt "* main" +assert_file_has_content summary.txt "* other" +assert_file_has_content summary.txt "* yet-another" +grep static-deltas summary.txt > static-deltas.txt +assert_file_has_content static-deltas.txt \ + $(${OSTREE} --repo=repo rev-parse origin:main) + +## Tests for handling of cached summaries while racing with remote summary updates + +# Make 2 different but valid summary/signature pairs to test races with +${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{,.1} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{,.1} +mkdir ${test_tmpdir}/ostree-srv/even-another-files +cd ${test_tmpdir}/ostree-srv/even-another-files +echo 'hello world even another object' > even-another-hello-world +${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo commit ${COMMIT_SIGN} -b even-another -s "A commit" -m "Another Commit body" +${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{,.2} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{,.2} +cd ${test_tmpdir} + +# Reset to the old valid summary and pull to cache it +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.1,} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.1,} +repo_reinit --set=verification-key=${PUBLIC_KEY} +${OSTREE} --repo=repo pull origin main +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.1 >&2 + +# Simulate a pull race where the client gets the old summary and the new +# summary signature since it was generated on the server between the +# requests +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} +if ${OSTREE} --repo=repo pull origin main 2>err.txt; then + assert_not_reached "Successful pull with old summary" +fi +assert_file_has_content err.txt "Can't verify summary" +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.1 >&2 + +# Publish correct summary and check that subsequent pull succeeds +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.2,} +${OSTREE} --repo=repo pull origin main +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.2 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.2 >&2 + +echo "ok ${engine} pull with signed summary remote old summary" + + +# Reset to the old valid summary and pull to cache it +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.1,} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.1,} +repo_reinit --set=verification-key=${PUBLIC_KEY} +${OSTREE} --repo=repo pull origin main +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.1 >&2 + +# Simulate a broken summary cache to see if it can be recovered from. +# Prior to commit c4c2b5eb the client would save the summary to the +# cache before validating the signature. That would mean the cache would +# have mismatched summary and signature and ostree would remain +# deadlocked there until the remote published a new signature. +# +# First pull with OSTREE_REPO_TEST_ERROR=invalid-cache to see the +# invalid cache is detected. Then pull again to check if it can be +# recovered from. +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.2 repo/tmp/cache/summaries/origin +if OSTREE_REPO_TEST_ERROR=invalid-cache ${OSTREE} --repo=repo pull origin main 2>err.txt; then + assert_not_reached "Should have hit OSTREE_REPO_TEST_ERROR_INVALID_CACHE" +fi +assert_file_has_content err.txt "OSTREE_REPO_TEST_ERROR_INVALID_CACHE" +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.2 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.1 >&2 +${OSTREE} --repo=repo pull origin main +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.1 >&2 + +# Publish new signature and check that subsequent pull succeeds +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.2,} +cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} +${OSTREE} --repo=repo pull origin main +assert_has_file repo/tmp/cache/summaries/origin +assert_has_file repo/tmp/cache/summaries/origin.sig +cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.2 >&2 +cmp repo/tmp/cache/summaries/origin.sig ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig.2 >&2 + +echo "ok ${engine} pull with signed summary broken cache" + From 65c16a8318b992b436fb666d675eb195fe3cfc71 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 26 Nov 2019 13:35:30 +0300 Subject: [PATCH 063/177] man: add signature options for ostree summary Add a description of new options `--sign-type` and `--sign` for `ostree summary` command. Signed-off-by: Denis Pynkin --- man/ostree-summary.xml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/man/ostree-summary.xml b/man/ostree-summary.xml index 387dacd7..8305b4e1 100644 --- a/man/ostree-summary.xml +++ b/man/ostree-summary.xml @@ -51,7 +51,7 @@ Boston, MA 02111-1307, USA. - ostree summary --gpg-sign=KEYID --gpg-homedir=HOMEDIR --update --add-metadata=KEY=VALUE + ostree summary --gpg-sign=KEYID --gpg-homedir=HOMEDIR --sign=KEYID --sign-type=ENGINE --update --add-metadata=KEY=VALUE @@ -139,6 +139,39 @@ Boston, MA 02111-1307, USA. + + =ENGINE + + Use particular signature engine. Currently + available ed25519 and dummy + signature types. + + The default is ed25519. + + + + + ="KEY-ID" + + There KEY-ID is: + + + + + base64-encoded secret key for commit signing. + + + + + + + ASCII-string used as secret key. + + + + + + From 809176b1ffa4d8b167f6fca26b7f697795eedcff Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 27 Nov 2019 12:21:39 +0300 Subject: [PATCH 064/177] gpg: do not fail GPG-related configuration get for remote We don't need anymore stubs for verification options for remotes in case if ostree built without GPG support. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 9 --------- src/libostree/ostree-repo.c | 18 ------------------ 2 files changed, 27 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index c6a94d72..203b34cf 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3986,7 +3986,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_free (pull_data->remote_name); pull_data->remote_name = g_strdup (remote_name_or_baseurl); -#ifndef OSTREE_DISABLE_GPGME /* Fetch GPG verification settings from remote if it wasn't already * explicitly set in the options. */ if (!opt_gpg_verify_set) @@ -3998,7 +3997,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, pull_data->remote_name, &pull_data->gpg_verify_summary, error)) goto out; -#endif /* OSTREE_DISABLE_GPGME */ /* Fetch verification settings from remote if it wasn't already * explicitly set in the options. */ if (!opt_sign_verify_set) @@ -6460,9 +6458,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autofree char *metalink_url_string = NULL; g_autoptr(GBytes) summary = NULL; g_autoptr(GBytes) signatures = NULL; -#ifndef OSTREE_DISABLE_GPGME gboolean gpg_verify_summary; -#endif gboolean ret = FALSE; gboolean summary_is_from_cache; @@ -6484,7 +6480,6 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, error)) goto out; -#ifndef OSTREE_DISABLE_GPGME if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) goto out; @@ -6538,10 +6533,6 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, } } -#else - g_message ("%s: GPG feature is disabled in a build time", __FUNCTION__); -#endif /* OSTREE_DISABLE_GPGME */ - if (out_summary != NULL) *out_summary = g_steal_pointer (&summary); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3aeecc5c..36fde3dc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2028,17 +2028,8 @@ ostree_repo_remote_get_gpg_verify (OstreeRepo *self, return TRUE; } -#ifndef OSTREE_DISABLE_GPGME return ostree_repo_get_remote_boolean_option (self, name, "gpg-verify", TRUE, out_gpg_verify, error); -#else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - if (out_gpg_verify != NULL) - *out_gpg_verify = FALSE; - return FALSE; -#endif /* OSTREE_DISABLE_GPGME */ } /** @@ -2060,17 +2051,8 @@ ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, gboolean *out_gpg_verify_summary, GError **error) { -#ifndef OSTREE_DISABLE_GPGME return ostree_repo_get_remote_boolean_option (self, name, "gpg-verify-summary", FALSE, out_gpg_verify_summary, error); -#else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - if (out_gpg_verify_summary != NULL) - *out_gpg_verify_summary = FALSE; - return FALSE; -#endif /* OSTREE_DISABLE_GPGME */ } /** From df36984684e84551f1db944a48af0978aee3469d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 27 Nov 2019 13:15:26 +0000 Subject: [PATCH 065/177] lib/repo-pull: change sign supporting functions Change the API of supporting functions `_load_public_keys()` and `_ostree_repo_sign_verify()` -- pass repo object and remote name instead of OtPullData object. This allows to use these functions not only in pull-related places. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 203b34cf..311e37d8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1483,8 +1483,9 @@ process_verify_result (OtPullData *pull_data, * Returns: %TRUE if no configuration or any key loaded. * */ static gboolean -_load_public_keys (OtPullData *pull_data, - OstreeSign *sign) +_load_public_keys (OstreeSign *sign, + OstreeRepo *repo, + const gchar *remote_name) { g_autofree gchar *pk_ascii = NULL; @@ -1493,13 +1494,13 @@ _load_public_keys (OtPullData *pull_data, gboolean loaded_inlined = TRUE; g_autoptr (GError) error = NULL; - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, + ostree_repo_get_remote_option (repo, + remote_name, "verification-file", NULL, &pk_file, NULL); - ostree_repo_get_remote_option (pull_data->repo, - pull_data->remote_name, + ostree_repo_get_remote_option (repo, + remote_name, "verification-key", NULL, &pk_ascii, NULL); @@ -1565,7 +1566,8 @@ _load_public_keys (OtPullData *pull_data, } static gboolean -_ostree_repo_sign_verify (OtPullData *pull_data, +_ostree_repo_sign_verify (OstreeRepo *repo, + const gchar *remote_name, GBytes *signed_data, GVariant *metadata) { @@ -1594,7 +1596,7 @@ _ostree_repo_sign_verify (OtPullData *pull_data, continue; /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (pull_data, sign)) + if (!_load_public_keys (sign, repo, remote_name)) continue; /* Return true if any signature fit to pre-loaded public keys. @@ -1656,7 +1658,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, return FALSE; } - if (!_ostree_repo_sign_verify (pull_data, signed_data, detached_metadata)) + if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata)) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Can't verify commit"); @@ -2012,7 +2014,7 @@ scan_commit_object (OtPullData *pull_data, continue; /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (pull_data, sign)) + if (!_load_public_keys (sign, pull_data->repo, pull_data->remote_name)) continue; /* Set return to true if any sign fit */ @@ -4406,7 +4408,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, bytes_sig, FALSE); - if (!_ostree_repo_sign_verify (pull_data, bytes_summary, signatures)) + if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures)) { gboolean ret = FALSE; @@ -4437,7 +4439,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (_ostree_repo_sign_verify (pull_data, bytes_summary, signatures)) + if (_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures)) ret = TRUE; } From c69dce3c314e534dbebfaff946f0ca170a3ad7a7 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 27 Nov 2019 13:24:01 +0000 Subject: [PATCH 066/177] lib/repo-pull: set default for sign-verify-summary Use FALSE as default for summary verification while pulling from remote. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 311e37d8..e666b0c8 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -6314,6 +6314,7 @@ ostree_repo_pull_from_remotes_async (OstreeRepo *self, #endif /* OSTREE_DISABLE_GPGME */ g_variant_dict_insert (&local_options_dict, "gpg-verify-summary", "b", FALSE); g_variant_dict_insert (&local_options_dict, "sign-verify", "b", FALSE); + g_variant_dict_insert (&local_options_dict, "sign-verify-summary", "b", FALSE); g_variant_dict_insert (&local_options_dict, "inherit-transaction", "b", TRUE); if (result->remote->refspec_name != NULL) g_variant_dict_insert (&local_options_dict, "override-remote-name", "s", result->remote->refspec_name); From b97ab81bab817c03cb047844b4b8518e45afaf65 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 27 Nov 2019 16:26:54 +0300 Subject: [PATCH 067/177] lib/repo-pull: add signature check while fetching summary Check the signature of downloaded summary file. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 81 ++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index e666b0c8..0c33b0f6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -6462,6 +6462,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GBytes) summary = NULL; g_autoptr(GBytes) signatures = NULL; gboolean gpg_verify_summary; + gboolean sign_verify_summary; gboolean ret = FALSE; gboolean summary_is_from_cache; @@ -6486,33 +6487,73 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, name, &gpg_verify_summary, error)) goto out; - if (gpg_verify_summary && summary == NULL) + if (gpg_verify_summary) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)"); - goto out; + if (summary == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no summary found (check that the configured URL in remote config is correct)"); + goto out; + } + + if (signatures == NULL) + { + g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, + "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } + + /* Verify any summary signatures. */ + if (summary != NULL && signatures != NULL) + { + g_autoptr(OstreeGpgVerifyResult) result = NULL; + + result = ostree_repo_verify_summary (self, + name, + summary, + signatures, + cancellable, + error); + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + goto out; + } } - if (gpg_verify_summary && signatures == NULL) - { - g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, - "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)"); + if (!ostree_repo_get_remote_boolean_option (self, name, "sign-verify-summary", + FALSE, &sign_verify_summary, error)) goto out; - } - /* Verify any summary signatures. */ - if (gpg_verify_summary && summary != NULL && signatures != NULL) + if (sign_verify_summary) { - g_autoptr(OstreeGpgVerifyResult) result = NULL; + if (summary == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Signature verification enabled, but no summary found (check that the configured URL in remote config is correct)"); + goto out; + } - result = ostree_repo_verify_summary (self, - name, - summary, - signatures, - cancellable, - error); - if (!ostree_gpg_verify_result_require_valid_signature (result, error)) - goto out; + if (signatures == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Signature verification enabled, but no summary signatures found (use sign-verify-summary=false in remote config to disable)"); + goto out; + } + + /* Verify any summary signatures. */ + if (summary != NULL && signatures != NULL) + { + g_autoptr(GVariant) sig_variant = NULL; + + sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + signatures, FALSE); + + if (!_ostree_repo_sign_verify (self, name, summary, sig_variant)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Signature verification enabled, but no valid signatures found"); + goto out; + } + } } if (!summary_is_from_cache && summary && signatures) From 8b3b35a04a72f50d09c4facf6fd5d51205e5408c Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 27 Nov 2019 16:46:24 +0300 Subject: [PATCH 068/177] bin/pull-local: add --sign-verify-summary Add option for enabling summary file verification while pulling from local. Signed-off-by: Denis Pynkin --- src/ostree/ot-builtin-pull-local.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 4dbd3bfd..695b09e5 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -40,6 +40,7 @@ static gboolean opt_require_static_deltas; static gboolean opt_gpg_verify; static gboolean opt_gpg_verify_summary; static gboolean opt_sign_verify; +static gboolean opt_sign_verify_summary; static int opt_depth = 0; /* ATTENTION: @@ -57,6 +58,7 @@ static GOptionEntry options[] = { { "gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify, "GPG verify commits (must specify --remote)", NULL }, { "gpg-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify_summary, "GPG verify summary (must specify --remote)", NULL }, { "sign-verify", 0, 0, G_OPTION_ARG_NONE, &opt_sign_verify, "Verify commits signature (must specify --remote)", NULL }, + { "sign-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_sign_verify, "Verify summary signature (must specify --remote)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -187,6 +189,9 @@ ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invoc if (opt_sign_verify) g_variant_builder_add (&builder, "{s@v}", "sign-verify", g_variant_new_variant (g_variant_new_boolean (TRUE))); + if (opt_sign_verify_summary) + g_variant_builder_add (&builder, "{s@v}", "sign-verify-summary", + g_variant_new_variant (g_variant_new_boolean (TRUE))); if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); From 4d0e3a66c5ec6cf1063a057a8e0ccf3d8be3d615 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 4 Dec 2019 20:40:55 +0000 Subject: [PATCH 069/177] lib/sign: make dummy engine non-public Remove unneeded public declaration for dummy signing engine. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 2 -- src/libostree/libostree-devel.sym | 1 - src/libostree/ostree-sign-dummy.c | 14 +++++++------- src/libostree/ostree-sign-dummy.h | 9 ++++----- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index a5a632a5..e61b46c3 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -709,7 +709,6 @@ ostree_kernel_args_to_string
ostree-sign OstreeSign -OstreeSignDummy OstreeSignEd25519 ostree_sign_list_names ostree_sign_commit @@ -726,6 +725,5 @@ ostree_sign_set_sk ostree_sign_summary ostree_sign_get_type -ostree_sign_dummy_get_type ostree_sign_ed25519_get_type
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 29221f5d..730eac3e 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -36,7 +36,6 @@ global: ostree_sign_add_pk; ostree_sign_set_sk; ostree_sign_summary; - ostree_sign_dummy_get_type; ostree_sign_ed25519_get_type; } LIBOSTREE_2020.1; diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 034c7d6b..df5a7b82 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -50,7 +50,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSignDummy, g_object_unref) static void ostree_sign_dummy_iface_init (OstreeSignInterface *self); -G_DEFINE_TYPE_WITH_CODE (OstreeSignDummy, ostree_sign_dummy, G_TYPE_OBJECT, +G_DEFINE_TYPE_WITH_CODE (OstreeSignDummy, _ostree_sign_dummy, G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE (OSTREE_TYPE_SIGN, ostree_sign_dummy_iface_init)); static void @@ -70,13 +70,13 @@ ostree_sign_dummy_iface_init (OstreeSignInterface *self) } static void -ostree_sign_dummy_class_init (OstreeSignDummyClass *self) +_ostree_sign_dummy_class_init (OstreeSignDummyClass *self) { g_debug ("%s enter", __FUNCTION__); } static void -ostree_sign_dummy_init (OstreeSignDummy *self) +_ostree_sign_dummy_init (OstreeSignDummy *self) { g_debug ("%s enter", __FUNCTION__); @@ -88,7 +88,7 @@ gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **err { g_debug ("%s enter", __FUNCTION__); - OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); g_free(sign->sk_ascii); @@ -101,7 +101,7 @@ gboolean ostree_sign_dummy_set_pk (OstreeSign *self, GVariant *key, GError **err { g_debug ("%s enter", __FUNCTION__); - OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); g_free(sign->pk_ascii); @@ -120,7 +120,7 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); *signature = g_bytes_new (sign->sk_ascii, strlen(sign->sk_ascii)); @@ -158,7 +158,7 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (data != NULL, FALSE); - OstreeSignDummy *sign = ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); + OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); gboolean ret = FALSE; diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index 6fc3a363..c37bcdfa 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -30,17 +30,16 @@ G_BEGIN_DECLS -#define OSTREE_TYPE_SIGN_DUMMY (ostree_sign_dummy_get_type ()) +#define OSTREE_TYPE_SIGN_DUMMY (_ostree_sign_dummy_get_type ()) -_OSTREE_PUBLIC -GType ostree_sign_dummy_get_type (void); +GType _ostree_sign_dummy_get_type (void); G_GNUC_BEGIN_IGNORE_DEPRECATIONS typedef struct _OstreeSignDummy OstreeSignDummy; typedef struct { GObjectClass parent_class; } OstreeSignDummyClass; -static inline OstreeSignDummy *OSTREE_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_sign_dummy_get_type (), OstreeSignDummy); } -static inline gboolean OSTREE_IS_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_sign_dummy_get_type ()); } +static inline OstreeSignDummy *OSTREE_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, _ostree_sign_dummy_get_type (), OstreeSignDummy); } +static inline gboolean OSTREE_IS_SIGN_DUMMY (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, _ostree_sign_dummy_get_type ()); } G_GNUC_END_IGNORE_DEPRECATIONS From b4050b4a34721f1302ade2f6a82a1148a243b46c Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 4 Dec 2019 20:42:52 +0000 Subject: [PATCH 070/177] lib/sign: make ed25519 engine non-public Remove unneeded public declaration for ed25519 signing engine. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 2 -- src/libostree/libostree-devel.sym | 1 - src/libostree/ostree-sign-ed25519.c | 18 +++++++++--------- src/libostree/ostree-sign-ed25519.h | 9 ++++----- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index e61b46c3..4dcdc482 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -709,7 +709,6 @@ ostree_kernel_args_to_string
ostree-sign OstreeSign -OstreeSignEd25519 ostree_sign_list_names ostree_sign_commit ostree_sign_commit_verify @@ -725,5 +724,4 @@ ostree_sign_set_sk ostree_sign_summary ostree_sign_get_type -ostree_sign_ed25519_get_type
diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 730eac3e..4348ab8d 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -36,7 +36,6 @@ global: ostree_sign_add_pk; ostree_sign_set_sk; ostree_sign_summary; - ostree_sign_ed25519_get_type; } LIBOSTREE_2020.1; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 2d5bdb16..d28a5c32 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -55,7 +55,7 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeSignEd25519, g_object_unref) static void ostree_sign_ed25519_iface_init (OstreeSignInterface *self); -G_DEFINE_TYPE_WITH_CODE (OstreeSignEd25519, ostree_sign_ed25519, G_TYPE_OBJECT, +G_DEFINE_TYPE_WITH_CODE (OstreeSignEd25519, _ostree_sign_ed25519, G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE (OSTREE_TYPE_SIGN, ostree_sign_ed25519_iface_init)); static void @@ -76,13 +76,13 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) } static void -ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) +_ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) { g_debug ("%s enter", __FUNCTION__); } static void -ostree_sign_ed25519_init (OstreeSignEd25519 *self) +_ostree_sign_ed25519_init (OstreeSignEd25519 *self) { g_debug ("%s enter", __FUNCTION__); @@ -111,7 +111,7 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); #ifdef HAVE_LIBSODIUM guchar *sig = NULL; @@ -163,7 +163,7 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, g_return_val_if_fail (data != NULL, FALSE); gboolean ret = FALSE; - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); if (signatures == NULL) { @@ -284,7 +284,7 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); #ifdef HAVE_LIBSODIUM - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); /* Clear secret key */ if (sign->secret_key != NULL) @@ -326,7 +326,7 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); #ifdef HAVE_LIBSODIUM - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); ostree_sign_ed25519_clear_keys (self, error); @@ -391,7 +391,7 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); #ifdef HAVE_LIBSODIUM - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); g_autofree char * hex = NULL; gpointer key = NULL; @@ -446,7 +446,7 @@ _ed25519_add_revoked (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - OstreeSignEd25519 *sign = ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); g_autofree char * hex = NULL; gpointer key = NULL; diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index 6e5dd665..76c7e14d 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -30,17 +30,16 @@ G_BEGIN_DECLS -#define OSTREE_TYPE_SIGN_ED25519 (ostree_sign_ed25519_get_type ()) +#define OSTREE_TYPE_SIGN_ED25519 (_ostree_sign_ed25519_get_type ()) -_OSTREE_PUBLIC -GType ostree_sign_ed25519_get_type (void); +GType _ostree_sign_ed25519_get_type (void); G_GNUC_BEGIN_IGNORE_DEPRECATIONS typedef struct _OstreeSignEd25519 OstreeSignEd25519; typedef struct { GObjectClass parent_class; } OstreeSignEd25519Class; -static inline OstreeSignEd25519 *OSTREE_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, ostree_sign_ed25519_get_type (), OstreeSignEd25519); } -static inline gboolean OSTREE_IS_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, ostree_sign_ed25519_get_type ()); } +static inline OstreeSignEd25519 *OSTREE_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_CAST (ptr, _ostree_sign_ed25519_get_type (), OstreeSignEd25519); } +static inline gboolean OSTREE_IS_SIGN_ED25519 (gpointer ptr) { return G_TYPE_CHECK_INSTANCE_TYPE (ptr, _ostree_sign_ed25519_get_type ()); } G_GNUC_END_IGNORE_DEPRECATIONS From dd27461e221748c745ad3e63f787cf3f8b98cdde Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 6 Dec 2019 15:04:14 +0300 Subject: [PATCH 071/177] lib/sign: better error handling of ed25519 initialization Add more precise error handling for ed25519 initialization. Check the initialization status at the beginning of every public function provided by ed25519 engine. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 98 ++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index d28a5c32..6bf0b10b 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -39,10 +39,17 @@ #define OSTREE_SIGN_METADATA_ED25519_KEY "ostree.sign.ed25519" #define OSTREE_SIGN_METADATA_ED25519_TYPE "aay" +typedef enum +{ + ED25519_OK, + ED25519_NOT_SUPPORTED, + ED25519_FAILED_INITIALIZATION +} ed25519_state; + struct _OstreeSignEd25519 { GObject parent; - gboolean initialized; + ed25519_state state; guchar *secret_key; GList *public_keys; GList *revoked_keys; @@ -86,22 +93,39 @@ _ostree_sign_ed25519_init (OstreeSignEd25519 *self) { g_debug ("%s enter", __FUNCTION__); - self->initialized = TRUE; + self->state = ED25519_OK; self->secret_key = NULL; self->public_keys = NULL; self->revoked_keys = NULL; #ifdef HAVE_LIBSODIUM if (sodium_init() < 0) - { - self->initialized = FALSE; - g_warning ("libsodium library couldn't be initialized"); - } + self->state = ED25519_FAILED_INITIALIZATION; #else - g_error ("ed25519 signature isn't supported"); + self->state = ED25519_NOT_SUPPORTED; #endif /* HAVE_LIBSODIUM */ } +static gboolean +_ostree_sign_ed25519_is_initialized (OstreeSignEd25519 *self, GError **error) +{ + switch (self->state) + { + case ED25519_OK: + break; + case ED25519_NOT_SUPPORTED: + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "ed25519: engine is not supported"); + return FALSE; + case ED25519_FAILED_INITIALIZATION: + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "ed25519: libsodium library isn't initialized properly"); + return FALSE; + } + + return TRUE; +} + gboolean ostree_sign_ed25519_data (OstreeSign *self, GBytes *data, GBytes **signature, @@ -117,10 +141,13 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, guchar *sig = NULL; #endif - if ((sign->initialized != TRUE) || (sign->secret_key == NULL)) + if (!_ostree_sign_ed25519_is_initialized (sign, error)) + goto err; + + if (sign->secret_key == NULL) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to sign: libsodium library isn't initialized properly"); + "secret key is not set"); goto err; } #ifdef HAVE_LIBSODIUM @@ -135,7 +162,7 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, sign->secret_key)) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to sign the object"); + "fail to sign the object"); goto err; } @@ -143,6 +170,7 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, return TRUE; #endif /* HAVE_LIBSODIUM */ err: + g_prefix_error (error, "Not able to sign: "); return FALSE; } @@ -165,11 +193,14 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + if (!_ostree_sign_ed25519_is_initialized (sign, error)) + goto out; + if (signatures == NULL) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: ed25519: commit have no signatures of my type"); + "ed25519: commit have no signatures of my type"); goto out; } @@ -177,14 +208,7 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: ed25519: wrong type passed for verification"); - goto out; - } - - if (sign->initialized != TRUE) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to verify: libsodium library isn't initialized properly"); + "ed25519: wrong type passed for verification"); goto out; } @@ -248,10 +272,12 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, if (ret != TRUE) g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not able to verify: no valid signatures found"); + "no valid signatures found"); #endif /* HAVE_LIBSODIUM */ out: + if (ret != TRUE) + g_prefix_error (error, "Not able to verify: "); return ret; } @@ -283,9 +309,12 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); -#ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + if (!_ostree_sign_ed25519_is_initialized (sign, error)) + goto err; + +#ifdef HAVE_LIBSODIUM /* Clear secret key */ if (sign->secret_key != NULL) { @@ -309,8 +338,9 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, } return TRUE; - #endif /* HAVE_LIBSODIUM */ + +err: return FALSE; } @@ -325,11 +355,13 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + + if (!ostree_sign_ed25519_clear_keys (self, error)) + goto err; + #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - ostree_sign_ed25519_clear_keys (self, error); - gsize n_elements = 0; if (g_variant_is_of_type (secret_key, G_VARIANT_TYPE_STRING)) @@ -357,9 +389,9 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, } return TRUE; +#endif /* HAVE_LIBSODIUM */ err: -#endif /* HAVE_LIBSODIUM */ return FALSE; } @@ -374,7 +406,8 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - ostree_sign_ed25519_clear_keys (self, error); + if (!ostree_sign_ed25519_clear_keys (self, error)) + return FALSE; return ostree_sign_ed25519_add_pk (self, public_key, error); } @@ -390,11 +423,14 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); -#ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + + if (!_ostree_sign_ed25519_is_initialized (sign, error)) + goto err; + +#ifdef HAVE_LIBSODIUM g_autofree char * hex = NULL; gpointer key = NULL; - gsize n_elements = 0; if (g_variant_is_of_type (public_key, G_VARIANT_TYPE_STRING)) @@ -430,9 +466,9 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, } return TRUE; +#endif /* HAVE_LIBSODIUM */ err: -#endif /* HAVE_LIBSODIUM */ return FALSE; } @@ -679,6 +715,10 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, const gchar *filename = NULL; + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); + if (!_ostree_sign_ed25519_is_initialized (sign, error)) + return FALSE; + /* Read keys only from single file provided */ if (g_variant_lookup (options, "filename", "&s", &filename)) return _load_pk_from_file (self, filename, TRUE, error); From 194ab368f2425bf2bd1d9d721927696b93eaf933 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 6 Dec 2019 17:18:04 +0300 Subject: [PATCH 072/177] lib/repo-pull: return error from signing engine Return the collected errors from signing engines in case if verification failed for the commit. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0c33b0f6..7bbb1acd 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2005,7 +2005,7 @@ scan_commit_object (OtPullData *pull_data, gboolean ret = FALSE; /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); - for (char **iter=names; iter && *iter; iter++) + for (char **iter=names; !ret && iter && *iter; iter++) { g_autoptr (OstreeSign) sign = NULL; g_autoptr (GError) local_error = NULL; @@ -2023,15 +2023,28 @@ scan_commit_object (OtPullData *pull_data, checksum, cancellable, &local_error)) - ret = TRUE; + { + ret = TRUE; + g_clear_error (error); + } + + /* Save error message for better reason detection later if needed */ + if (!ret) + { + if (*error == NULL) + g_propagate_error (error, g_steal_pointer (&local_error)); + else + g_prefix_error (error, "%s; ", local_error->message); + } + } - } if (!ret) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't verify commit"); + g_prefix_error (error, "Can't verify commit %s: ", checksum); return FALSE; } + /* Clear non-fatal error */ + g_clear_error (error); } /* If we found a legacy transaction flag, assume we have to scan. From 59b9e64b724e3e9337e8f47fe8f99448cd7086f7 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Sat, 7 Dec 2019 19:28:41 +0300 Subject: [PATCH 073/177] lib/repo-pull: return errors from signature engines Improve error handling for signatures checks -- passthrough real reasons from signature engines instead of using common messages. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 121 ++++++++++++++++++------------- 1 file changed, 70 insertions(+), 51 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7bbb1acd..08534398 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1485,14 +1485,14 @@ process_verify_result (OtPullData *pull_data, static gboolean _load_public_keys (OstreeSign *sign, OstreeRepo *repo, - const gchar *remote_name) + const gchar *remote_name, + GError **error) { g_autofree gchar *pk_ascii = NULL; g_autofree gchar *pk_file = NULL; gboolean loaded_from_file = TRUE; gboolean loaded_inlined = TRUE; - g_autoptr (GError) error = NULL; ostree_repo_get_remote_option (repo, remote_name, @@ -1528,48 +1528,53 @@ _load_public_keys (OstreeSign *sign, g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); options = g_variant_builder_end (builder); - if (ostree_sign_load_pk (sign, options, &error)) + if (ostree_sign_load_pk (sign, options, error)) loaded_from_file = TRUE; else - { - g_assert (error); g_debug("Unable to load public keys for '%s' from file '%s': %s", - ostree_sign_get_name(sign), pk_file, error->message); - g_clear_error (&error); - } + ostree_sign_get_name(sign), pk_file, (*error)->message); } if (pk_ascii != NULL) { + g_autoptr (GError) local_error = NULL; g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); /* Add inlined public key */ if (loaded_from_file) - loaded_inlined = ostree_sign_add_pk (sign, pk, &error); + loaded_inlined = ostree_sign_add_pk (sign, pk, &local_error); else - loaded_inlined = ostree_sign_set_pk (sign, pk, &error); + loaded_inlined = ostree_sign_set_pk (sign, pk, &local_error); if (!loaded_inlined) { - if (error == NULL) - g_set_error_literal (&error, G_IO_ERROR, G_IO_ERROR_FAILED, - "unknown reason"); - g_debug("Unable to load public key '%s' for '%s': %s", - pk_ascii, ostree_sign_get_name(sign), error->message); - g_clear_error (&error); + pk_ascii, ostree_sign_get_name(sign), local_error->message); + + /* Save error message for better reason detection later if needed */ + if (*error == NULL) + g_propagate_error (error, g_steal_pointer (&local_error)); + else + g_prefix_error (error, "%s; ", local_error->message); } } /* Return true if able to load from any source */ - return (loaded_from_file || loaded_inlined); + if (loaded_from_file || loaded_inlined) + { + g_clear_error (error); + return TRUE; + } + + return FALSE; } static gboolean _ostree_repo_sign_verify (OstreeRepo *repo, const gchar *remote_name, GBytes *signed_data, - GVariant *metadata) + GVariant *metadata, + GError **error) { /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); @@ -1596,18 +1601,33 @@ _ostree_repo_sign_verify (OstreeRepo *repo, continue; /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (sign, repo, remote_name)) - continue; + if (_load_public_keys (sign, repo, remote_name, &local_error)) + { + /* Return true if any signature fit to pre-loaded public keys. + * If no keys configured -- then system configuration will be used */ + if (ostree_sign_data_verify (sign, + signed_data, + signatures, + &local_error)) + { + g_clear_error (error); + return TRUE; + } + } - /* Return true if any signature fit to pre-loaded public keys. - * If no keys configured -- then system configuration will be used */ - if (ostree_sign_data_verify (sign, - signed_data, - signatures, - &local_error)) - return TRUE; + /* Save error message for better reason detection later if needed */ + if (*error == NULL) + g_propagate_error (error, g_steal_pointer (&local_error)); + else + g_prefix_error (error, "%s; ", local_error->message); } + /* In case if there were no signatures of known type + * or metadata contains invalid data */ + if (*error == NULL) + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Metadata doesn't signed with supported type"); + return FALSE; } @@ -1658,10 +1678,9 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, return FALSE; } - if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata)) + if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't verify commit"); + g_prefix_error (error, "Can't verify commit"); return FALSE; } @@ -2014,18 +2033,19 @@ scan_commit_object (OtPullData *pull_data, continue; /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (sign, pull_data->repo, pull_data->remote_name)) - continue; - - /* Set return to true if any sign fit */ - if (ostree_sign_commit_verify (sign, - pull_data->repo, - checksum, - cancellable, - &local_error)) + if (_load_public_keys (sign, pull_data->repo, pull_data->remote_name, &local_error)) { - ret = TRUE; - g_clear_error (error); + + /* Set return to true if any sign fit */ + if (ostree_sign_commit_verify (sign, + pull_data->repo, + checksum, + cancellable, + &local_error)) + { + ret = TRUE; + g_clear_error (error); + } } /* Save error message for better reason detection later if needed */ @@ -4416,12 +4436,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (bytes_summary && bytes_sig) { g_autoptr(GVariant) signatures = NULL; + g_autoptr(GError) temp_error = NULL; signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, bytes_sig, FALSE); - if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures)) + if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, &temp_error)) { gboolean ret = FALSE; @@ -4452,14 +4473,16 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures)) + if (_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) ret = TRUE; } + else + g_propagate_error (error, g_steal_pointer (&temp_error)); + if (!ret) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't verify summary"); + g_prefix_error (error, "Can't verify summary: "); goto out; } } @@ -6560,12 +6583,8 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, signatures, FALSE); - if (!_ostree_repo_sign_verify (self, name, summary, sig_variant)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Signature verification enabled, but no valid signatures found"); - goto out; - } + if (!_ostree_repo_sign_verify (self, name, summary, sig_variant, error)) + goto out; } } From fbc5927d7ef4ab552efe94abc4e0e69a113ba0e7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 29 Jan 2020 14:09:00 +0000 Subject: [PATCH 074/177] build-sys: Print libsodium status at end of configure Like we do with other features. --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index e41ccb70..5d9d2a23 100644 --- a/configure.ac +++ b/configure.ac @@ -641,6 +641,7 @@ echo " cryptographic checksums: $with_crypto systemd: $with_libsystemd libmount: $with_libmount + libsodium (ed25519 signatures): $with_libsodium libarchive (parse tar files directly): $with_libarchive static deltas: yes (always enabled now) O_TMPFILE: $enable_otmpfile From 2a0edccbd346fccb3f40102d4c7bfa3f22a14feb Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 29 Jan 2020 14:17:19 +0000 Subject: [PATCH 075/177] sign-ed25519: Convert some functions to new style The "new style" code generally avoids `goto err` because it conflicts with `__attribute__((cleanup))`. This fixes a compiler warning. --- src/libostree/ostree-sign-ed25519.c | 48 ++++++++--------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 6bf0b10b..2941a9fe 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -426,10 +426,9 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); if (!_ostree_sign_ed25519_is_initialized (sign, error)) - goto err; + return FALSE; #ifdef HAVE_LIBSODIUM - g_autofree char * hex = NULL; gpointer key = NULL; gsize n_elements = 0; @@ -444,20 +443,14 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, } else { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unknown ed25519 public key type"); - goto err; + return glnx_throw (error, "Unknown ed25519 public key type"); } - hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_autofree char *hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); if (n_elements != crypto_sign_PUBLICKEYBYTES) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Incorrect ed25519 public key"); - goto err; - } + return glnx_throw (error, "Incorrect ed25519 public key"); if (g_list_find_custom (sign->public_keys, key, _compare_ed25519_keys) == NULL) { @@ -465,11 +458,8 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, sign->public_keys = g_list_prepend (sign->public_keys, newkey); } - return TRUE; #endif /* HAVE_LIBSODIUM */ - -err: - return FALSE; + return TRUE; } #ifdef HAVE_LIBSODIUM @@ -482,32 +472,21 @@ _ed25519_add_revoked (OstreeSign *self, g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); + if (!g_variant_is_of_type (revoked_key, G_VARIANT_TYPE_STRING)) + return glnx_throw (error, "Unknown ed25519 revoked key type"); + OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); - g_autofree char * hex = NULL; - gpointer key = NULL; + const gchar *rk_ascii = g_variant_get_string (revoked_key, NULL); gsize n_elements = 0; + gpointer key = g_base64_decode (rk_ascii, &n_elements); - if (g_variant_is_of_type (revoked_key, G_VARIANT_TYPE_STRING)) - { - const gchar *rk_ascii = g_variant_get_string (revoked_key, NULL); - key = g_base64_decode (rk_ascii, &n_elements); - } - else - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unknown ed25519 revoked key type"); - goto err; - } - - hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_autofree char * hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); g_debug ("Read ed25519 revoked key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); if (n_elements != crypto_sign_PUBLICKEYBYTES) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Incorrect ed25519 revoked key"); - goto err; + return glnx_throw (error, "Incorrect ed25519 revoked key"); } if (g_list_find_custom (sign->revoked_keys, key, _compare_ed25519_keys) == NULL) @@ -517,9 +496,6 @@ _ed25519_add_revoked (OstreeSign *self, } return TRUE; - -err: - return FALSE; } #endif /* HAVE_LIBSODIUM */ From e2bd2abc67f1a4781c866aa1e69e6b7a447a5aee Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Feb 2020 14:54:00 +0000 Subject: [PATCH 076/177] sign-dummy: Convert to current code style This keeps the code style consistent. --- src/libostree/ostree-sign-dummy.c | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index df5a7b82..ef2caa9f 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -160,25 +160,13 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); - gboolean ret = FALSE; - if (signatures == NULL) - { - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: dummy: commit have no signatures of my type"); - goto err; - } - + return glnx_throw (error, "signature: dummy: commit have no signatures of my type"); if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_DUMMY_TYPE)) - { - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: dummy: wrong type passed for verification"); - goto err; - } + return glnx_throw (error, "signature: dummy: wrong type passed for verification"); + gboolean verified = FALSE; for (gsize i = 0; i < g_variant_n_children(signatures); i++) { g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); @@ -191,13 +179,12 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, g_debug("Stored signature %d: %s", (gint)i, sign->pk_ascii); if (!g_strcmp0(sign_ascii, sign->pk_ascii)) - ret = TRUE; + verified = TRUE; + else + return glnx_throw (error, "signature: dummy: incorrect signature %" G_GSIZE_FORMAT, i); } - if (ret == FALSE && *error == NULL) - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: dummy: incorrect signature"); + if (!verified) + return glnx_throw (error, "signature: dummy: no signatures"); -err: - return ret; + return TRUE; } From 9d02199675a5fff44716854fbbe5bd9874d0c6f9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 14 Feb 2020 15:52:48 +0000 Subject: [PATCH 077/177] signing: Remove g_debug(__FUNCTION__) This type of thing is better done via `gdb` and/or userspace tracing (systemtap/bpftrace etc.) --- src/libostree/ostree-sign-dummy.c | 10 ---------- src/libostree/ostree-sign-ed25519.c | 17 ----------------- src/libostree/ostree-sign.c | 16 ---------------- 3 files changed, 43 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index ef2caa9f..b9d8abf4 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -56,7 +56,6 @@ G_DEFINE_TYPE_WITH_CODE (OstreeSignDummy, _ostree_sign_dummy, G_TYPE_OBJECT, static void ostree_sign_dummy_iface_init (OstreeSignInterface *self) { - g_debug ("%s enter", __FUNCTION__); self->get_name = ostree_sign_dummy_get_name; self->data = ostree_sign_dummy_data; @@ -72,13 +71,11 @@ ostree_sign_dummy_iface_init (OstreeSignInterface *self) static void _ostree_sign_dummy_class_init (OstreeSignDummyClass *self) { - g_debug ("%s enter", __FUNCTION__); } static void _ostree_sign_dummy_init (OstreeSignDummy *self) { - g_debug ("%s enter", __FUNCTION__); self->sk_ascii = NULL; self->pk_ascii = NULL; @@ -86,7 +83,6 @@ _ostree_sign_dummy_init (OstreeSignDummy *self) gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **error) { - g_debug ("%s enter", __FUNCTION__); OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); @@ -99,7 +95,6 @@ gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **err gboolean ostree_sign_dummy_set_pk (OstreeSign *self, GVariant *key, GError **error) { - g_debug ("%s enter", __FUNCTION__); OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); @@ -117,7 +112,6 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); @@ -129,7 +123,6 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, const gchar * ostree_sign_dummy_get_name (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); return OSTREE_SIGN_DUMMY_NAME; @@ -137,14 +130,12 @@ const gchar * ostree_sign_dummy_get_name (OstreeSign *self) const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); return OSTREE_SIGN_METADATA_DUMMY_KEY; } const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); return OSTREE_SIGN_METADATA_DUMMY_TYPE; } @@ -154,7 +145,6 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, GVariant *signatures, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (data != NULL, FALSE); diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 2941a9fe..ce656678 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -68,7 +68,6 @@ G_DEFINE_TYPE_WITH_CODE (OstreeSignEd25519, _ostree_sign_ed25519, G_TYPE_OBJECT, static void ostree_sign_ed25519_iface_init (OstreeSignInterface *self) { - g_debug ("%s enter", __FUNCTION__); self->data = ostree_sign_ed25519_data; self->data_verify = ostree_sign_ed25519_data_verify; @@ -85,13 +84,11 @@ ostree_sign_ed25519_iface_init (OstreeSignInterface *self) static void _ostree_sign_ed25519_class_init (OstreeSignEd25519Class *self) { - g_debug ("%s enter", __FUNCTION__); } static void _ostree_sign_ed25519_init (OstreeSignEd25519 *self) { - g_debug ("%s enter", __FUNCTION__); self->state = ED25519_OK; self->secret_key = NULL; @@ -133,7 +130,6 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); @@ -186,7 +182,6 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, GVariant *signatures, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (data != NULL, FALSE); gboolean ret = FALSE; @@ -283,7 +278,6 @@ out: const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); return OSTREE_SIGN_ED25519_NAME; @@ -291,14 +285,12 @@ const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) const gchar * ostree_sign_ed25519_metadata_key (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); return OSTREE_SIGN_METADATA_ED25519_KEY; } const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); return OSTREE_SIGN_METADATA_ED25519_TYPE; } @@ -306,7 +298,6 @@ const gchar * ostree_sign_ed25519_metadata_format (OstreeSign *self) gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); @@ -352,7 +343,6 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -403,7 +393,6 @@ gboolean ostree_sign_ed25519_set_pk (OstreeSign *self, GVariant *public_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (!ostree_sign_ed25519_clear_keys (self, error)) @@ -420,7 +409,6 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, GVariant *public_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); @@ -469,7 +457,6 @@ _ed25519_add_revoked (OstreeSign *self, GVariant *revoked_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (!g_variant_is_of_type (revoked_key, G_VARIANT_TYPE_STRING)) @@ -506,7 +493,6 @@ _load_pk_from_stream (OstreeSign *self, gboolean trusted, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (key_data_in, FALSE); #ifdef HAVE_LIBSODIUM gboolean ret = FALSE; @@ -558,7 +544,6 @@ _load_pk_from_file (OstreeSign *self, gboolean trusted, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_debug ("Processing file '%s'", filename); g_autoptr (GFile) keyfile = NULL; @@ -602,7 +587,6 @@ _ed25519_load_pk (OstreeSign *self, gboolean trusted, GError **error) { - g_debug ("%s enter", __FUNCTION__); gboolean ret = FALSE; const gchar *custom_dir = NULL; @@ -687,7 +671,6 @@ ostree_sign_ed25519_load_pk (OstreeSign *self, GVariant *options, GError **error) { - g_debug ("%s enter", __FUNCTION__); const gchar *filename = NULL; diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index e7962425..23ec84ee 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -98,7 +98,6 @@ ostree_sign_default_init (OstreeSignInterface *iface) const gchar * ostree_sign_metadata_key (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_key != NULL, NULL); return OSTREE_SIGN_GET_IFACE (self)->metadata_key (self); @@ -119,7 +118,6 @@ ostree_sign_metadata_key (OstreeSign *self) const gchar * ostree_sign_metadata_format (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->metadata_format != NULL, NULL); return OSTREE_SIGN_GET_IFACE (self)->metadata_format (self); @@ -140,7 +138,6 @@ gboolean ostree_sign_clear_keys (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->clear_keys == NULL) return TRUE; @@ -168,7 +165,6 @@ ostree_sign_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->set_sk == NULL) return TRUE; @@ -197,7 +193,6 @@ ostree_sign_set_pk (OstreeSign *self, GVariant *public_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->set_pk == NULL) return TRUE; @@ -226,7 +221,6 @@ ostree_sign_add_pk (OstreeSign *self, GVariant *public_key, GError **error) { - g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->add_pk == NULL) return TRUE; @@ -266,7 +260,6 @@ ostree_sign_load_pk (OstreeSign *self, GVariant *options, GError **error) { - g_debug ("%s enter", __FUNCTION__); if (OSTREE_SIGN_GET_IFACE (self)->load_pk == NULL) return TRUE; @@ -300,7 +293,6 @@ ostree_sign_data (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data != NULL, FALSE); @@ -331,7 +323,6 @@ ostree_sign_data_verify (OstreeSign *self, GVariant *signatures, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data_verify != NULL, FALSE); @@ -346,7 +337,6 @@ _sign_detached_metadata_append (OstreeSign *self, GVariant *existing_metadata, GBytes *signature_bytes) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (signature_bytes != NULL, FALSE); GVariantDict metadata_dict; @@ -401,7 +391,6 @@ ostree_sign_commit_verify (OstreeSign *self, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_autoptr(GVariant) commit_variant = NULL; @@ -455,7 +444,6 @@ ostree_sign_commit_verify (OstreeSign *self, const gchar * ostree_sign_get_name (OstreeSign *self) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), NULL); g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->get_name != NULL, NULL); @@ -487,7 +475,6 @@ ostree_sign_commit (OstreeSign *self, GCancellable *cancellable, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_autoptr(GBytes) commit_data = NULL; g_autoptr(GBytes) signature = NULL; @@ -537,7 +524,6 @@ ostree_sign_commit (OstreeSign *self, GStrv ostree_sign_list_names(void) { - g_debug ("%s enter", __FUNCTION__); GStrv names = g_new0 (char *, G_N_ELEMENTS(sign_types) + 1); gint i = 0; @@ -567,7 +553,6 @@ ostree_sign_list_names(void) OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error) { - g_debug ("%s enter", __FUNCTION__); OstreeSign *sign = NULL; @@ -616,7 +601,6 @@ ostree_sign_summary (OstreeSign *self, GCancellable *cancellable, GError **error) { - g_debug ("%s enter", __FUNCTION__); g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); From 09d5b475afdb19000203ec090b8077b7cfc4eb44 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 31 Jan 2020 13:00:59 +0300 Subject: [PATCH 078/177] tests/sign: added check with file and single key on pull Additional test of signatures check behavior during the pull with keys file containing wrong signatures and correct verification key. Both are set as a part of remote's configuration. Signed-off-by: Denis Pynkin --- tests/test-signed-pull.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 4d8d7aab..e9d1bd75 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..6" +echo "1..8" setup_fake_remote_repo1 "archive" @@ -73,6 +73,8 @@ test_signed_pull "dummy" "" if ! has_libsodium; then echo "ok ed25519-key pull signed commit # SKIP due libsodium unavailability" echo "ok ed25519-key re-pull signature for stored commit # SKIP due libsodium unavailability" + echo "ok ed25519-key+file pull signed commit # SKIP due libsodium unavailability" + echo "ok ed25519-key+file re-pull signature for stored commit # SKIP due libsodium unavailability" echo "ok ed25519-file pull signed commit # SKIP due libsodium unavailability" echo "ok ed25519-file re-pull signature for stored commit # SKIP due libsodium unavailability" exit 0 @@ -98,6 +100,11 @@ for((i=0;i<100;i++)); do # Generate a list with some public signatures gen_ed25519_random_public done > ${PUBKEYS} + +# Test case with the file containing incorrect signatures and with the correct key set +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "${PUBKEYS}" +test_signed_pull "ed25519" "key+file" + # Add correct key into the list echo ${PUBLIC} >> ${PUBKEYS} From aaf73f6afca4c72ef9c7be6f64507babe7130a1b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 18 Feb 2020 00:11:38 +0300 Subject: [PATCH 079/177] sign-ed25519: Convert functions to new style The "new style" code generally avoids `goto err` because it conflicts with `__attribute__((cleanup))`. This fixes a compiler warning. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 118 ++++++++-------------------- 1 file changed, 34 insertions(+), 84 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index ce656678..8df61aed 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -111,13 +111,9 @@ _ostree_sign_ed25519_is_initialized (OstreeSignEd25519 *self, GError **error) case ED25519_OK: break; case ED25519_NOT_SUPPORTED: - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "ed25519: engine is not supported"); - return FALSE; + return glnx_throw(error, "ed25519: engine is not supported"); case ED25519_FAILED_INITIALIZATION: - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "ed25519: libsodium library isn't initialized properly"); - return FALSE; + return glnx_throw(error, "ed25519: libsodium library isn't initialized properly"); } return TRUE; @@ -138,14 +134,11 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, #endif if (!_ostree_sign_ed25519_is_initialized (sign, error)) - goto err; + return FALSE; if (sign->secret_key == NULL) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "secret key is not set"); - goto err; - } + return glnx_throw (error, "Not able to sign: secret key is not set"); + #ifdef HAVE_LIBSODIUM unsigned long long sig_size = 0; @@ -157,16 +150,12 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, g_bytes_get_size (data), sign->secret_key)) { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "fail to sign the object"); - goto err; + return glnx_throw (error, "Not able to sign: fail to sign the object"); } *signature = g_bytes_new_take (sig, sig_size); return TRUE; #endif /* HAVE_LIBSODIUM */ -err: - g_prefix_error (error, "Not able to sign: "); return FALSE; } @@ -184,28 +173,17 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, { g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (data != NULL, FALSE); - gboolean ret = FALSE; OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); if (!_ostree_sign_ed25519_is_initialized (sign, error)) - goto out; + return FALSE; if (signatures == NULL) - { - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "ed25519: commit have no signatures of my type"); - goto out; - } + return glnx_throw (error, "ed25519: commit have no signatures of my type"); if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_ED25519_TYPE)) - { - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "ed25519: wrong type passed for verification"); - goto out; - } + return glnx_throw (error, "ed25519: wrong type passed for verification"); #ifdef HAVE_LIBSODIUM /* If no keys pre-loaded then, @@ -219,7 +197,7 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, options = g_variant_builder_end (builder); if (!ostree_sign_ed25519_load_pk (self, options, error)) - goto out; + return FALSE; } g_debug ("verify: data hash = 0x%x", g_bytes_hash(data)); @@ -257,23 +235,17 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, } else { - ret = TRUE; g_debug ("Signature verified successfully with key '%s'", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); - break; + return TRUE; } } } - if (ret != TRUE) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "no valid signatures found"); + return glnx_throw (error, "Not able to verify: no valid signatures found"); #endif /* HAVE_LIBSODIUM */ -out: - if (ret != TRUE) - g_prefix_error (error, "Not able to verify: "); - return ret; + return FALSE; } const gchar * ostree_sign_ed25519_get_name (OstreeSign *self) @@ -303,7 +275,7 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); if (!_ostree_sign_ed25519_is_initialized (sign, error)) - goto err; + return FALSE; #ifdef HAVE_LIBSODIUM /* Clear secret key */ @@ -331,7 +303,6 @@ gboolean ostree_sign_ed25519_clear_keys (OstreeSign *self, return TRUE; #endif /* HAVE_LIBSODIUM */ -err: return FALSE; } @@ -347,7 +318,7 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, if (!ostree_sign_ed25519_clear_keys (self, error)) - goto err; + return FALSE; #ifdef HAVE_LIBSODIUM OstreeSignEd25519 *sign = _ostree_sign_ed25519_get_instance_private(OSTREE_SIGN_ED25519(self)); @@ -365,23 +336,15 @@ gboolean ostree_sign_ed25519_set_sk (OstreeSign *self, } else { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Unknown ed25519 secret key type"); - goto err; + return glnx_throw (error, "Unknown ed25519 secret key type"); } - if (n_elements != crypto_sign_SECRETKEYBYTES) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Incorrect ed25519 secret key"); - goto err; - } + return glnx_throw (error, "Incorrect ed25519 secret key"); return TRUE; #endif /* HAVE_LIBSODIUM */ -err: return FALSE; } @@ -506,10 +469,10 @@ _load_pk_from_stream (OstreeSign *self, gboolean added = FALSE; if (*error != NULL) - goto err; + return FALSE; if (line == NULL) - goto out; + return ret; /* Read the key itself */ /* base64 encoded key */ @@ -529,11 +492,6 @@ _load_pk_from_stream (OstreeSign *self, if (added) ret = TRUE; } - -out: - return ret; - -err: #endif /* HAVE_LIBSODIUM */ return FALSE; } @@ -553,15 +511,13 @@ _load_pk_from_file (OstreeSign *self, if (!g_file_test (filename, G_FILE_TEST_IS_REGULAR)) { g_debug ("Can't open file '%s' with public keys", filename); - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "File object '%s' is not a regular file", filename); - goto err; + return glnx_throw (error, "File object '%s' is not a regular file", filename); } keyfile = g_file_new_for_path (filename); key_stream_in = g_file_read (keyfile, NULL, error); if (key_stream_in == NULL) - goto err; + return FALSE; key_data_in = g_data_input_stream_new (G_INPUT_STREAM(key_stream_in)); g_assert (key_data_in != NULL); @@ -569,16 +525,12 @@ _load_pk_from_file (OstreeSign *self, if (!_load_pk_from_stream (self, key_data_in, trusted, error)) { if (error == NULL || *error == NULL) - g_set_error (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: ed25519: no valid keys in file '%s'", - filename); - goto err; + return glnx_throw (error, + "signature: ed25519: no valid keys in file '%s'", + filename); } return TRUE; -err: - return FALSE; } static gboolean @@ -639,21 +591,19 @@ _ed25519_load_pk (OstreeSign *self, /* Scan all well-known files */ for (gint i=0; i < ed25519_files->len; i++) { - if (!_load_pk_from_file (self, (gchar *)g_ptr_array_index (ed25519_files, i), trusted, error)) - { - g_debug ("Problem with loading ed25519 %s keys from `%s`", - trusted ? "public" : "revoked", - (gchar *)g_ptr_array_index (ed25519_files, i)); - g_clear_error(error); - } - else - ret = TRUE; + if (!_load_pk_from_file (self, (gchar *)g_ptr_array_index (ed25519_files, i), trusted, error)) + { + g_debug ("Problem with loading ed25519 %s keys from `%s`", + trusted ? "public" : "revoked", + (gchar *)g_ptr_array_index (ed25519_files, i)); + g_clear_error(error); + } + else + ret = TRUE; } if (!ret && (error == NULL || *error == NULL)) - g_set_error_literal (error, - G_IO_ERROR, G_IO_ERROR_FAILED, - "signature: ed25519: no keys loaded"); + return glnx_throw (error, "signature: ed25519: no keys loaded"); return ret; } From 1e3bdef2851260b8ac3503744d4157d527e3823d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 18 Feb 2020 00:46:51 +0300 Subject: [PATCH 080/177] sign-dummy: optimize ostree_sign_dummy_data_verify Return TRUE as soon as any signature verified. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-dummy.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index b9d8abf4..722d461b 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -156,7 +156,6 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, if (!g_variant_is_of_type (signatures, (GVariantType *) OSTREE_SIGN_METADATA_DUMMY_TYPE)) return glnx_throw (error, "signature: dummy: wrong type passed for verification"); - gboolean verified = FALSE; for (gsize i = 0; i < g_variant_n_children(signatures); i++) { g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); @@ -169,12 +168,10 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, g_debug("Stored signature %d: %s", (gint)i, sign->pk_ascii); if (!g_strcmp0(sign_ascii, sign->pk_ascii)) - verified = TRUE; + return TRUE; else return glnx_throw (error, "signature: dummy: incorrect signature %" G_GSIZE_FORMAT, i); } - if (!verified) - return glnx_throw (error, "signature: dummy: no signatures"); - return TRUE; + return glnx_throw (error, "signature: dummy: no signatures"); } From e16faa58f4c3be33f073b8986cd628f18047dabd Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 18 Feb 2020 00:50:21 +0300 Subject: [PATCH 081/177] lib/sign: convert ostree_sign_summary to new style The "new style" code generally avoids `goto err` because it conflicts with `__attribute__((cleanup))`. This fixes a compiler warning. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign.c | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 23ec84ee..c96f751f 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -604,37 +604,32 @@ ostree_sign_summary (OstreeSign *self, g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (OSTREE_IS_REPO (repo), FALSE); - gboolean ret = FALSE; - g_autoptr(GVariant) normalized = NULL; g_autoptr(GBytes) summary_data = NULL; g_autoptr(GVariant) metadata = NULL; glnx_autofd int fd = -1; if (!glnx_openat_rdonly (repo->repo_dir_fd, "summary", TRUE, &fd, error)) - goto out; + return FALSE; summary_data = ot_fd_readall_or_mmap (fd, 0, error); if (!summary_data) - goto out; + return FALSE; /* Note that fd is reused below */ glnx_close_fd (&fd); if (!ot_openat_ignore_enoent (repo->repo_dir_fd, "summary.sig", &fd, error)) - goto out; + return FALSE; + if (fd >= 0) { if (!ot_variant_read_fd (fd, 0, OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, FALSE, &metadata, error)) - goto out; + return FALSE; } if (g_variant_n_children(keys) == 0) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No keys passed for signing summary"); - goto out; - } + return glnx_throw (error, "No keys passed for signing summary"); GVariantIter *iter; GVariant *key; @@ -645,14 +640,14 @@ ostree_sign_summary (OstreeSign *self, g_autoptr (GBytes) signature = NULL; if (!ostree_sign_set_sk (self, key, error)) - goto out; + return FALSE; if (!ostree_sign_data (self, summary_data, &signature, cancellable, error)) - goto out; + return FALSE; g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata); metadata = @@ -667,10 +662,7 @@ ostree_sign_summary (OstreeSign *self, g_variant_get_data (normalized), g_variant_get_size (normalized), cancellable, error)) - goto out; + return FALSE; - ret = TRUE; - -out: - return ret; + return TRUE; } From 5a39281fbec6956e16ef17f33288f6d6b7ceb205 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 20 Feb 2020 02:43:36 +0300 Subject: [PATCH 082/177] tests/sign: check pull failure with invalid remote options Pull should to fail if no known signature available in remote's configuration or well-known places. Signed-off-by: Denis Pynkin --- tests/test-signed-pull.sh | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index e9d1bd75..f222db4f 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..8" +echo "1..11" setup_fake_remote_repo1 "archive" @@ -67,6 +67,31 @@ function test_signed_pull() { DUMMYSIGN="dummysign" COMMIT_ARGS="--sign=${DUMMYSIGN} --sign-type=dummy" repo_init --set=sign-verify=true + +# Check if verification-key and verification-file options throw error with wrong keys +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} \ + -b main -s "A signed commit" --tree=ref=main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +if ${CMD_PREFIX} ostree --repo=repo pull origin main; then + assert_not_reached "pull without keys unexpectedly succeeded" +fi +echo "ok pull failure without keys preloaded" + +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "somewrongkey" +if ${CMD_PREFIX} ostree --repo=repo pull origin main; then + assert_not_reached "pull with unknown key unexpectedly succeeded" +fi +echo "ok pull failure with incorrect key option" + +${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-key +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "/non/existing/file" +if ${CMD_PREFIX} ostree --repo=repo pull origin main; then + assert_not_reached "pull with unknown keys file unexpectedly succeeded" +fi +echo "ok pull failure with incorrect keys file option" + +# Test with correct dummy key ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${DUMMYSIGN}" test_signed_pull "dummy" "" From 584ad405491a3839d1478420bc0788077cb9e7f3 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 20 Feb 2020 03:24:14 +0300 Subject: [PATCH 083/177] lib/sign: return false for non-implemented functions Do not mask implementation anymore since we have a working engines integrated with pulling mechanism. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index c96f751f..281fabc8 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -138,9 +138,9 @@ gboolean ostree_sign_clear_keys (OstreeSign *self, GError **error) { - + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->clear_keys == NULL) - return TRUE; + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->clear_keys (self, error); } @@ -165,9 +165,9 @@ ostree_sign_set_sk (OstreeSign *self, GVariant *secret_key, GError **error) { - + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->set_sk == NULL) - return TRUE; + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->set_sk (self, secret_key, error); } @@ -193,9 +193,9 @@ ostree_sign_set_pk (OstreeSign *self, GVariant *public_key, GError **error) { - + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->set_pk == NULL) - return TRUE; + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->set_pk (self, public_key, error); } @@ -221,9 +221,9 @@ ostree_sign_add_pk (OstreeSign *self, GVariant *public_key, GError **error) { - + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->add_pk == NULL) - return TRUE; + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->add_pk (self, public_key, error); } @@ -260,9 +260,9 @@ ostree_sign_load_pk (OstreeSign *self, GVariant *options, GError **error) { - + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->load_pk == NULL) - return TRUE; + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->load_pk (self, options, error); } @@ -294,7 +294,8 @@ ostree_sign_data (OstreeSign *self, { g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data != NULL, FALSE); + if (OSTREE_SIGN_GET_IFACE (self)->data == NULL) + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->data (self, data, signature, cancellable, error); } @@ -324,7 +325,8 @@ ostree_sign_data_verify (OstreeSign *self, GError **error) { g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); - g_return_val_if_fail (OSTREE_SIGN_GET_IFACE (self)->data_verify != NULL, FALSE); + if (OSTREE_SIGN_GET_IFACE (self)->data_verify == NULL) + return glnx_throw (error, "not implemented"); return OSTREE_SIGN_GET_IFACE (self)->data_verify(self, data, signatures, error); } From cce3864160438511fe767444f682f7ab9fb510c1 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 20 Feb 2020 03:59:05 +0300 Subject: [PATCH 084/177] sign-pull: improve error handling Use glnx_* functions in signature related pull code for clear error handling. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 92 +++++++++++-------------------- tests/test-signed-pull-summary.sh | 4 +- 2 files changed, 34 insertions(+), 62 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 08534398..043e8dee 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1470,7 +1470,7 @@ process_verify_result (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ -/* _remote_load_public_keys: +/* _load_public_keys: * * Load public keys according remote's configuration: * inlined key passed via config option `verification-key` or @@ -1493,6 +1493,9 @@ _load_public_keys (OstreeSign *sign, g_autofree gchar *pk_file = NULL; gboolean loaded_from_file = TRUE; gboolean loaded_inlined = TRUE; + g_autoptr (GError) verification_error = NULL; + + glnx_throw (&verification_error, "no public keys loaded"); ostree_repo_get_remote_option (repo, remote_name, @@ -1521,6 +1524,7 @@ _load_public_keys (OstreeSign *sign, if (pk_file != NULL) { + g_autoptr (GError) local_error = NULL; g_autoptr (GVariantBuilder) builder = NULL; g_autoptr (GVariant) options = NULL; @@ -1528,11 +1532,15 @@ _load_public_keys (OstreeSign *sign, g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); options = g_variant_builder_end (builder); - if (ostree_sign_load_pk (sign, options, error)) + if (ostree_sign_load_pk (sign, options, &local_error)) loaded_from_file = TRUE; else + { g_debug("Unable to load public keys for '%s' from file '%s': %s", - ostree_sign_get_name(sign), pk_file, (*error)->message); + ostree_sign_get_name(sign), pk_file, local_error->message); + /* Save error message for better reason detection later if needed */ + glnx_prefix_error (&verification_error, "%s", local_error->message); + } } if (pk_ascii != NULL) @@ -1549,24 +1557,18 @@ _load_public_keys (OstreeSign *sign, if (!loaded_inlined) { g_debug("Unable to load public key '%s' for '%s': %s", - pk_ascii, ostree_sign_get_name(sign), local_error->message); + pk_ascii, ostree_sign_get_name (sign), local_error->message); /* Save error message for better reason detection later if needed */ - if (*error == NULL) - g_propagate_error (error, g_steal_pointer (&local_error)); - else - g_prefix_error (error, "%s; ", local_error->message); + glnx_prefix_error (&verification_error, "%s", local_error->message); } } /* Return true if able to load from any source */ if (loaded_from_file || loaded_inlined) - { - g_clear_error (error); - return TRUE; - } + return TRUE; - return FALSE; + return glnx_throw (error, "%s", verification_error->message); } static gboolean @@ -1578,6 +1580,10 @@ _ostree_repo_sign_verify (OstreeRepo *repo, { /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); + g_autoptr (GError) verification_error = NULL; + + glnx_throw (&verification_error, "signed with unknown key"); + for (char **iter=names; iter && *iter; iter++) { g_autoptr (OstreeSign) sign = NULL; @@ -1609,26 +1615,16 @@ _ostree_repo_sign_verify (OstreeRepo *repo, signed_data, signatures, &local_error)) - { - g_clear_error (error); - return TRUE; - } + return TRUE; } /* Save error message for better reason detection later if needed */ - if (*error == NULL) - g_propagate_error (error, g_steal_pointer (&local_error)); - else - g_prefix_error (error, "%s; ", local_error->message); + glnx_prefix_error (&verification_error, "%s", local_error->message); } /* In case if there were no signatures of known type * or metadata contains invalid data */ - if (*error == NULL) - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Metadata doesn't signed with supported type"); - - return FALSE; + return glnx_throw (error, "%s", verification_error->message); } static gboolean @@ -1672,17 +1668,10 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, { /* Nothing to check if detached metadata is absent */ if (detached_metadata == NULL) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't verify commit without detached metadata"); - return FALSE; - } + return glnx_throw (error, "Can't verify commit without detached metadata"); if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) - { - g_prefix_error (error, "Can't verify commit"); - return FALSE; - } + return glnx_prefix_error (error, "Can't verify commit"); /* Mark the commit as verified to avoid double verification * see process_verify_result () for rationale */ @@ -2022,6 +2011,8 @@ scan_commit_object (OtPullData *pull_data, !g_hash_table_contains (pull_data->verified_commits, checksum)) { gboolean ret = FALSE; + g_autoptr (GError) verification_error = NULL; + /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); for (char **iter=names; !ret && iter && *iter; iter++) @@ -2042,29 +2033,16 @@ scan_commit_object (OtPullData *pull_data, checksum, cancellable, &local_error)) - { - ret = TRUE; - g_clear_error (error); - } + ret = TRUE; } /* Save error message for better reason detection later if needed */ if (!ret) - { - if (*error == NULL) - g_propagate_error (error, g_steal_pointer (&local_error)); - else - g_prefix_error (error, "%s; ", local_error->message); - } + glnx_prefix_error (&verification_error, "%s", local_error->message); } if (!ret) - { - g_prefix_error (error, "Can't verify commit %s: ", checksum); - return FALSE; - } - /* Clear non-fatal error */ - g_clear_error (error); + return glnx_throw (error, "Can't verify commit %s: %s", checksum, verification_error->message); } /* If we found a legacy transaction flag, assume we have to scan. @@ -4444,8 +4422,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, &temp_error)) { - gboolean ret = FALSE; - if (summary_from_cache) { /* The cached summary doesn't match, fetch a new one and verify again */ @@ -4473,16 +4449,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) - ret = TRUE; + if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) + goto out; } else - g_propagate_error (error, g_steal_pointer (&temp_error)); - - - if (!ret) { - g_prefix_error (error, "Can't verify summary: "); + g_propagate_error (error, g_steal_pointer (&temp_error)); goto out; } } diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh index 7b18cf65..ee731e86 100755 --- a/tests/test-signed-pull-summary.sh +++ b/tests/test-signed-pull-summary.sh @@ -155,7 +155,7 @@ do if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with invalid ${engine} signature" fi - assert_file_has_content err.txt "Can't verify summary" + assert_file_has_content err.txt "signed with unknown key" mv ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.good,} echo "ok ${engine} pull with invalid ${engine} summary signature fails" @@ -223,7 +223,7 @@ cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with old summary" fi -assert_file_has_content err.txt "Can't verify summary" +assert_file_has_content err.txt "signed with unknown key" assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 From e2c601687b81330f863b59bc42503532c1527ec5 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Mon, 23 Mar 2020 15:54:06 +0300 Subject: [PATCH 085/177] ostree-repo: improve error handling Correctly return "error" from `ostree_repo_sign_commit()` in case if GPG is not enabled. Use glnx_* functions in signature related pull code for clear error handling if GPG isn't enabled. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo.c | 38 +++++++++---------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 36fde3dc..b4a42b6b 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2326,10 +2326,7 @@ out: return ret; #else /* OSTREE_DISABLE_GPGME */ - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - return FALSE; + return glnx_throw (error, "GPG feature is disabled in a build time"); #endif /* OSTREE_DISABLE_GPGME */ } @@ -4974,10 +4971,7 @@ ostree_repo_append_gpg_signature (OstreeRepo *self, return TRUE; #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - return FALSE; + return glnx_throw (error, "GPG feature is disabled in a build time"); #endif /* OSTREE_DISABLE_GPGME */ } @@ -5129,7 +5123,7 @@ ostree_repo_sign_commit (OstreeRepo *self, return TRUE; #else /* FIXME: Return false until refactoring */ - return FALSE; + return glnx_throw (error, "GPG feature is disabled in a build time"); #endif /* OSTREE_DISABLE_GPGME */ } @@ -5221,10 +5215,7 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, return TRUE; #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - return FALSE; + return glnx_throw (error, "GPG feature is disabled in a build time"); #endif /* OSTREE_DISABLE_GPGME */ } @@ -5498,10 +5489,7 @@ ostree_repo_verify_commit (OstreeRepo *self, return TRUE; #else /* FIXME: Return false until refactoring */ - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); - return FALSE; + return glnx_throw (error, "GPG feature is disabled in a build time"); #endif /* OSTREE_DISABLE_GPGME */ } @@ -5536,9 +5524,7 @@ ostree_repo_verify_commit_ext (OstreeRepo *self, cancellable, error); #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); + glnx_throw (error, "GPG feature is disabled in a build time"); return NULL; #endif /* OSTREE_DISABLE_GPGME */ } @@ -5575,9 +5561,7 @@ ostree_repo_verify_commit_for_remote (OstreeRepo *self, cancellable, error); #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); + glnx_throw (error, "GPG feature is disabled in a build time"); return NULL; #endif /* OSTREE_DISABLE_GPGME */ } @@ -5627,9 +5611,7 @@ ostree_repo_gpg_verify_data (OstreeRepo *self, cancellable, error); #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); + glnx_throw (error, "GPG feature is disabled in a build time"); return NULL; #endif /* OSTREE_DISABLE_GPGME */ } @@ -5675,9 +5657,7 @@ ostree_repo_verify_summary (OstreeRepo *self, cancellable, error); #else - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", - __FUNCTION__); + glnx_throw (error, "GPG feature is disabled in a build time"); return NULL; #endif /* OSTREE_DISABLE_GPGME */ } From 9cb148c9b960ddf704132aa3bf0ac6e2deaeefd5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 28 Mar 2020 14:07:00 +0000 Subject: [PATCH 086/177] lib: Add error prefixing for sysroot load and repo open Noticed this while writing tests for a core `ostree_sysroot_load()` entrypoint. And decided to do the same for `ostree_repo_open()`, and while there also noted we had a duplicate error prefixing for the open (more recently `glnx_opendirat()` automatically prefixes with the path). --- src/libostree/ostree-repo.c | 7 +++---- src/libostree/ostree-sysroot.c | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 3aeecc5c..40251aa2 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3257,6 +3257,8 @@ ostree_repo_open (OstreeRepo *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("opening repo", error); + struct stat stbuf; g_return_val_if_fail (error == NULL || *error == NULL, FALSE); @@ -3291,10 +3293,7 @@ ostree_repo_open (OstreeRepo *self, g_assert (self->repodir); if (!glnx_opendirat (AT_FDCWD, gs_file_get_path_cached (self->repodir), TRUE, &self->repo_dir_fd, error)) - { - g_prefix_error (error, "%s: ", gs_file_get_path_cached (self->repodir)); - return FALSE; - } + return FALSE; } if (!glnx_fstat (self->repo_dir_fd, &stbuf, error)) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 55a61014..7fbedee6 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1003,6 +1003,7 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, GCancellable *cancellable, GError **error) { + GLNX_AUTO_PREFIX_ERROR ("loading sysroot", error); if (!ostree_sysroot_initialize (self, error)) return FALSE; From cd37293b5ad46b6158f46537b1535ae373e10cbe Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 13 Nov 2019 18:29:45 -0800 Subject: [PATCH 087/177] find-remotes: Add a --mirror option This will be useful in the unit test added by the next commit. It just passes OSTREE_REPO_PULL_FLAGS_MIRROR to the call to ostree_repo_pull_from_remotes_async(). --- man/ostree-find-remotes.xml | 10 ++++++++++ src/ostree/ot-builtin-find-remotes.c | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/man/ostree-find-remotes.xml b/man/ostree-find-remotes.xml index b3796c31..89e278d7 100644 --- a/man/ostree-find-remotes.xml +++ b/man/ostree-find-remotes.xml @@ -116,6 +116,16 @@ Boston, MA 02111-1307, USA. + + + + + Do a mirror pull (see the documentation for + ostree pull --mirror). This option can + only be used in combination with . + + + diff --git a/src/ostree/ot-builtin-find-remotes.c b/src/ostree/ot-builtin-find-remotes.c index f255501a..944533ca 100644 --- a/src/ostree/ot-builtin-find-remotes.c +++ b/src/ostree/ot-builtin-find-remotes.c @@ -35,6 +35,7 @@ static gchar *opt_cache_dir = NULL; static gchar *opt_finders = NULL; static gboolean opt_disable_fsync = FALSE; static gboolean opt_pull = FALSE; +static gboolean opt_mirror = FALSE; static GOptionEntry options[] = { @@ -42,6 +43,7 @@ static GOptionEntry options[] = { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "finders", 0, 0, G_OPTION_ARG_STRING, &opt_finders, "Use the specified comma separated list of finders (e.g. config,lan,mount)", "FINDERS" }, { "pull", 0, 0, G_OPTION_ARG_NONE, &opt_pull, "Pull the updates after finding them", NULL }, + { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Do a mirror pull (see ostree pull --mirror)", NULL}, { NULL } }; @@ -188,6 +190,7 @@ ostree_builtin_find_remotes (int argc, g_auto(OstreeRepoFinderResultv) results = NULL; g_auto(GLnxConsoleRef) console = { 0, }; g_autoptr(GHashTable) refs_found = NULL; /* set (element-type OstreeCollectionRef) */ + g_autoptr(GVariant) pull_options = NULL; context = g_option_context_new ("COLLECTION-ID REF [COLLECTION-ID REF...]"); @@ -210,6 +213,12 @@ ostree_builtin_find_remotes (int argc, return FALSE; } + if (opt_mirror && !opt_pull) + { + ot_util_usage_error (context, "When --mirror is specified, --pull must also be", error); + return FALSE; + } + if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); @@ -359,13 +368,24 @@ ostree_builtin_find_remotes (int argc, if (!opt_pull) return TRUE; + { + GVariantBuilder builder; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + + if (opt_mirror) + g_variant_builder_add (&builder, "{s@v}", "flags", + g_variant_new_variant (g_variant_new_int32 (OSTREE_REPO_PULL_FLAGS_MIRROR))); + + pull_options = g_variant_ref_sink (g_variant_builder_end (&builder)); + } + /* Run the pull operation. */ if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); ostree_repo_pull_from_remotes_async (repo, (const OstreeRepoFinderResult * const *) results, - NULL, /* no options */ + pull_options, progress, cancellable, get_result_cb, &pull_result); From 164b2aa35bc2e39615f461decc90381d9bedd132 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 13 Nov 2019 18:30:58 -0800 Subject: [PATCH 088/177] Don't copy summary for collection-ref mirror subset pulls When we're only pulling a subset of the refs available in the remote, it doesn't make sense to copy the remote's summary (which may not be valid for the local repo). This makes the check here match the one done several lines above when we decide whether to error out if there's no remote summary available. This extends the fix in https://github.com/ostreedev/ostree/pull/935 for the case of collection-refs. Also, add a unit test for this issue, based on the existing one in pull-test.sh. --- src/libostree/ostree-repo-pull.c | 3 ++- tests/test-pull-collections.sh | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 381cce47..a0f6db02 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4567,7 +4567,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - if (pull_data->is_mirror && pull_data->summary_data && !refs_to_fetch && !configured_branches) + if (pull_data->is_mirror && pull_data->summary_data && + !refs_to_fetch && !opt_collection_refs_set && !configured_branches) { GLnxFileReplaceFlags replaceflag = pull_data->repo->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : 0; diff --git a/tests/test-pull-collections.sh b/tests/test-pull-collections.sh index d1de5f88..cd60ab21 100755 --- a/tests/test-pull-collections.sh +++ b/tests/test-pull-collections.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo '1..6' +echo '1..7' cd ${test_tmpdir} @@ -258,3 +258,15 @@ then fi echo "ok 6 pull refs from local repos" + +ostree_repo_init local-mirror +do_remote_add local-mirror collection-repo --collection-id org.example.CollectionRepo +# Generate a summary in the local mirror; don't use do_summary to avoid gpg +${CMD_PREFIX} ostree --repo=local-mirror summary --update +summarysig=$(sha256sum < local-mirror/summary | cut -f 1 -d ' ') +# Mirror subset of refs: A collection-ref version of https://github.com/ostreedev/ostree/issues/846 +${CMD_PREFIX} ostree --repo=local-mirror find-remotes --pull --mirror --finders=config org.example.CollectionRepo goodcref1 +newsummarysig=$(sha256sum < local-mirror/summary | cut -f 1 -d ' ') +assert_streq ${summarysig} ${newsummarysig} + +echo "ok 7 mirror pull subset of collection-refs with summary" From 40a60f1e83f28c0f7fde5ae11708a915b4c0730c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 22 Jan 2020 19:44:31 +0000 Subject: [PATCH 089/177] sysroot: Reorganize sysroot load code a bit No functional changes; prep for a future patch which will load the "live" deployment rather than reading the bootloader configs. --- src/libostree/ostree-sysroot.c | 106 ++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 7fbedee6..0dc7a392 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -988,31 +988,12 @@ _ostree_sysroot_reload_staged (OstreeSysroot *self, return TRUE; } -/** - * ostree_sysroot_load_if_changed: - * @self: #OstreeSysroot - * @out_changed: (out caller-allocates): - * @cancellable: Cancellable - * @error: Error - * - * Since: 2016.4 - */ -gboolean -ostree_sysroot_load_if_changed (OstreeSysroot *self, - gboolean *out_changed, - GCancellable *cancellable, - GError **error) +static gboolean +sysroot_load_from_bootloader_configs (OstreeSysroot *self, + GCancellable *cancellable, + GError **error) { - GLNX_AUTO_PREFIX_ERROR ("loading sysroot", error); - if (!ostree_sysroot_initialize (self, error)) - return FALSE; - - /* Here we also lazily initialize the repository. We didn't do this - * previous to v2017.6, but we do now to support the error-free - * ostree_sysroot_repo() API. - */ - if (!ensure_repo (self, error)) - return FALSE; + struct stat stbuf; int bootversion = 0; if (!read_current_bootversion (self, &bootversion, cancellable, error)) @@ -1023,27 +1004,6 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, cancellable, error)) return FALSE; - struct stat stbuf; - if (!glnx_fstatat (self->sysroot_fd, "ostree/deploy", &stbuf, 0, error)) - return FALSE; - - if (out_changed) - { - if (self->loaded_ts.tv_sec == stbuf.st_mtim.tv_sec && - self->loaded_ts.tv_nsec == stbuf.st_mtim.tv_nsec) - { - *out_changed = FALSE; - /* Note early return */ - return TRUE; - } - } - - g_clear_pointer (&self->deployments, g_ptr_array_unref); - g_clear_object (&self->booted_deployment); - g_clear_object (&self->staged_deployment); - self->bootversion = -1; - self->subbootversion = -1; - g_autoptr(GPtrArray) boot_loader_configs = NULL; if (!_ostree_sysroot_read_boot_loader_configs (self, bootversion, &boot_loader_configs, cancellable, error)) @@ -1121,8 +1081,60 @@ ostree_sysroot_load_if_changed (OstreeSysroot *self, self->bootversion = bootversion; self->subbootversion = subbootversion; - self->deployments = deployments; - deployments = NULL; /* Transfer ownership */ + self->deployments = g_steal_pointer (&deployments); + + return TRUE; +} + +/** + * ostree_sysroot_load_if_changed: + * @self: #OstreeSysroot + * @out_changed: (out caller-allocates): + * @cancellable: Cancellable + * @error: Error + * + * Since: 2016.4 + */ +gboolean +ostree_sysroot_load_if_changed (OstreeSysroot *self, + gboolean *out_changed, + GCancellable *cancellable, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR ("loading sysroot", error); + + if (!ostree_sysroot_initialize (self, error)) + return FALSE; + + /* Here we also lazily initialize the repository. We didn't do this + * previous to v2017.6, but we do now to support the error-free + * ostree_sysroot_repo() API. + */ + if (!ensure_repo (self, error)) + return FALSE; + + struct stat stbuf; + if (!glnx_fstatat (self->sysroot_fd, "ostree/deploy", &stbuf, 0, error)) + return FALSE; + + if (self->loaded_ts.tv_sec == stbuf.st_mtim.tv_sec && + self->loaded_ts.tv_nsec == stbuf.st_mtim.tv_nsec) + { + if (out_changed) + *out_changed = FALSE; + /* Note early return */ + return TRUE; + } + + g_clear_pointer (&self->deployments, g_ptr_array_unref); + g_clear_object (&self->booted_deployment); + g_clear_object (&self->staged_deployment); + self->bootversion = -1; + self->subbootversion = -1; + + if (!sysroot_load_from_bootloader_configs (self, cancellable, error)) + return FALSE; + self->loaded_ts = stbuf.st_mtim; if (out_changed) From 7091e1b5ca7b78573fb42a73cff657c109349b11 Mon Sep 17 00:00:00 2001 From: Felix Krull Date: Wed, 1 Apr 2020 23:06:19 +0200 Subject: [PATCH 090/177] lib: fix typo in function docs --- src/libostree/ostree-repo-finder-avahi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c index f8d1f878..1b085ea0 100644 --- a/src/libostree/ostree-repo-finder-avahi.c +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -1358,7 +1358,7 @@ ostree_repo_finder_avahi_init (OstreeRepoFinderAvahi *self) } /** - * ostree_repo-finder_avahi_new: + * ostree_repo_finder_avahi_new: * @context: (transfer none) (nullable): a #GMainContext for processing Avahi * events in, or %NULL to use the current thread-default * From c4d519733d2f7e78fc9e83ffb6418ea62f11395c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 4 Apr 2020 17:05:56 +0000 Subject: [PATCH 091/177] lib: Squash two gtk-doc warnings Just noticed these while doing a build. --- src/libostree/ostree-repo-deprecated.h | 2 ++ src/libostree/ostree-repo.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-deprecated.h b/src/libostree/ostree-repo-deprecated.h index 4265aa97..1c9d2251 100644 --- a/src/libostree/ostree-repo-deprecated.h +++ b/src/libostree/ostree-repo-deprecated.h @@ -24,9 +24,11 @@ #include "ostree-core.h" #include "ostree-types.h" +#ifndef __GI_SCANNER__ #ifndef G_GNUC_DEPRECATED_FOR # define G_GNUC_DEPRECATED_FOR(x) #endif +#endif G_BEGIN_DECLS diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 014afff9..1027ac77 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -694,7 +694,7 @@ void ostree_repo_commit_modifier_set_sepolicy (OstreeRepoCommitModifier _OSTREE_PUBLIC gboolean ostree_repo_commit_modifier_set_sepolicy_from_commit (OstreeRepoCommitModifier *modifier, OstreeRepo *repo, - const char *commit, + const char *rev, GCancellable *cancellable, GError **error); From 6d89b969ed4edc5f2f982508e7d925f77d9c3cba Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 5 Apr 2020 18:37:54 +0000 Subject: [PATCH 092/177] tests/pull-sizes: Disable xattrs everywhere I am getting SELinux xattrs committed in local development (inside https://github.com/cgwalters/coretoolbox ), which throws off the hardcoded sizes in this test. --- tests/test-pull-sizes.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test-pull-sizes.sh b/tests/test-pull-sizes.sh index 8ee07cc8..1ce0a736 100755 --- a/tests/test-pull-sizes.sh +++ b/tests/test-pull-sizes.sh @@ -23,6 +23,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh +# Committing SELinux attributes throws off the hardcoded sizes below +export OSTREE_NO_XATTRS=1 setup_fake_remote_repo1 "archive" "--generate-sizes" echo '1..3' From 47539874b8e5259040a11a79b3c155e633fcae34 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 5 Apr 2020 18:22:49 +0000 Subject: [PATCH 093/177] pull: Update key loading function to match error style This code wasn't written with idiomatic GError usage; it's not standard to construct an error up front and continually append to its message. The exit from a function is usually `return TRUE`, with error conditions before that. Updating it to match style reveals what I think is a bug; we were silently ignoring failure to parse key files. --- src/libostree/ostree-repo-pull.c | 23 +++++++---------------- tests/test-signed-pull.sh | 1 + 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 454eaf8a..a5e3a729 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1488,14 +1488,10 @@ _load_public_keys (OstreeSign *sign, const gchar *remote_name, GError **error) { - g_autofree gchar *pk_ascii = NULL; g_autofree gchar *pk_file = NULL; gboolean loaded_from_file = TRUE; gboolean loaded_inlined = TRUE; - g_autoptr (GError) verification_error = NULL; - - glnx_throw (&verification_error, "no public keys loaded"); ostree_repo_get_remote_option (repo, remote_name, @@ -1536,10 +1532,8 @@ _load_public_keys (OstreeSign *sign, loaded_from_file = TRUE; else { - g_debug("Unable to load public keys for '%s' from file '%s': %s", - ostree_sign_get_name(sign), pk_file, local_error->message); - /* Save error message for better reason detection later if needed */ - glnx_prefix_error (&verification_error, "%s", local_error->message); + return glnx_throw (error, "Failed loading '%s' keys from '%s", + ostree_sign_get_name (sign), pk_file); } } @@ -1556,19 +1550,16 @@ _load_public_keys (OstreeSign *sign, if (!loaded_inlined) { - g_debug("Unable to load public key '%s' for '%s': %s", - pk_ascii, ostree_sign_get_name (sign), local_error->message); - - /* Save error message for better reason detection later if needed */ - glnx_prefix_error (&verification_error, "%s", local_error->message); + return glnx_throw (error, "Failed loading '%s' keys from inline `verification-key`", + ostree_sign_get_name (sign)); } } /* Return true if able to load from any source */ - if (loaded_from_file || loaded_inlined) - return TRUE; + if (!(loaded_from_file || loaded_inlined)) + return glnx_throw (error, "No keys found"); - return glnx_throw (error, "%s", verification_error->message); + return TRUE; } static gboolean diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index f222db4f..2c677d46 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -93,6 +93,7 @@ echo "ok pull failure with incorrect keys file option" # Test with correct dummy key ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${DUMMYSIGN}" +${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-file test_signed_pull "dummy" "" if ! has_libsodium; then From 768eee87f667397a5f714eeed071a6ebe9686d46 Mon Sep 17 00:00:00 2001 From: Javier Martinez Canillas Date: Thu, 26 Mar 2020 17:20:08 +0100 Subject: [PATCH 094/177] grub2: Don't add menu entries if GRUB supports parsing BLS snippets This is another attempt to avoid having duplicated menu entries caused by GRUB having support to parse BLS snippets and the 15_ostree script adding menu entries as well. The previous attempt was in commit 985a1410029 ("grub2: Exit gracefully if the configuration has BLS enabled") but that lead to users not having menu entries at all, due having an old GRUB version that was not able to parse the BLS snippets. This happened because the GRUB bootloader is never updated in the ESP as a part of the OSTree upgrade transaction. The logic is similar to the previous commit, the 15_ostree script exits if able to determine that the bootloader can parse the BLS snippets directly. But this time it will not only check that a BLS configuration was enabled, but also that a /boot/grub2/.grub2-blscfg-supported file exists. This file has to be created by a component outside of OSTree that also takes care of updating GRUB to a version that has proper BLS support. --- src/boot/grub2/grub2-15_ostree | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/boot/grub2/grub2-15_ostree b/src/boot/grub2/grub2-15_ostree index 0b9bf930..9042708e 100644 --- a/src/boot/grub2/grub2-15_ostree +++ b/src/boot/grub2/grub2-15_ostree @@ -26,6 +26,16 @@ if ! test -d /ostree/repo; then exit 0 fi +# Gracefully exit if the grub2 configuration has BLS enabled, +# and the installed version has support for the blscfg module. +# Since there is no need to create menu entries for that case. +# See: https://src.fedoraproject.org/rpms/grub2/c/7c2bab5e98d +. /etc/default/grub +if test -f /boot/grub2/.grub2-blscfg-supported && \ + test "${GRUB_ENABLE_BLSCFG}" = "true"; then + exit 0 +fi + # Make sure we're in the right environment if ! test -n "${GRUB_DEVICE}"; then echo "This script must be run as a child of grub2-mkconfig" 1>&2 From 329a82c57e954392a2b33e60bcb8163892064205 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 5 Apr 2020 22:23:39 +0000 Subject: [PATCH 095/177] commit: Add --base argument I was trying to followup the `--selinux-policy-from-base` work to add a `cosa build --fast=overlay` for coreos-assembler, but hit on the fact that using e.g. `--owner-uid` disables commit optimizations. A while ago, https://github.com/ostreedev/ostree/pull/1643 landed which optimized this for the case where no modifications are provided. But, we really need the SELinux policy bits, and it's super convenient to run `ostree commit` as non-root. It's fairly surprising actually that it's taken us so long to iterate on a good interface for this "commit changes on top of a base" model. In practice, many nontrivial cases really end up needing to do a (hardlink) checkout, and that case is optimized. But for this coreos-assembler work I want to directly overlay onto a commit object another commit object. That previous PR above added exactly the API we need, so let's expose it in the CLI. What you can see happening in the test is that we provide `--owner-uid 42`, but that only applies to directories/files that were added in the commit. And now that I look at this, I think what we really want here is to avoid changing directories that exist in the base, but eh; in practice the main use here is for `--owner-uid 0` while committing as non-root; and that works fine with this since the baseline uid will be zero as well. --- man/ostree-commit.xml | 9 +++++++++ src/ostree/ot-builtin-commit.c | 29 ++++++++++++++++++++++++++++- tests/archive-test.sh | 8 ++++++++ tests/test-archivez.sh | 2 +- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index 2c821fc1..b0c5b335 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -106,6 +106,15 @@ Boston, MA 02111-1307, USA. + + ="REV" + + + Start from the content in a commit. This differs from --tree=ref=REV in that no commit modifiers are applied. This is usually what you want when + creating a derived commit. This is also used for --selinux-policy-from-base. + + + ="KEY=VALUE" diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 606af2be..72fa2841 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -58,6 +58,7 @@ static gboolean opt_selinux_policy_from_base; static gboolean opt_canonical_permissions; static gboolean opt_consume; static gboolean opt_devino_canonical; +static char *opt_base; static char **opt_trees; static gint opt_owner_uid = -1; static gint opt_owner_gid = -1; @@ -101,6 +102,7 @@ static GOptionEntry options[] = { { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, { "no-bindings", 0, 0, G_OPTION_ARG_NONE, &opt_no_bindings, "Do not write any ref bindings", NULL }, { "bind-ref", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_bind_refs, "Add a ref to ref binding commit metadata", "BRANCH" }, + { "base", 0, 0, G_OPTION_ARG_STRING, &opt_base, "Start from the given commit as a base (no modifiers apply)" }, { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "dir=PATH or tar=TARFILE or ref=COMMIT" }, { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Add a key/value pair to metadata", "KEY=VALUE" }, { "add-metadata", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_variants, "Add a key/value pair to metadata, where the KEY is a string, an VALUE is g_variant_parse() formatted", "KEY=VALUE" }, @@ -600,7 +602,32 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio if (opt_link_checkout_speedup && !ostree_repo_scan_hardlinks (repo, cancellable, error)) goto out; - mtree = ostree_mutable_tree_new (); + if (opt_base) + { + g_autofree char *base_commit = NULL; + g_autoptr(GFile) root = NULL; + if (!ostree_repo_read_commit (repo, opt_base, &root, &base_commit, cancellable, error)) + goto out; + OstreeRepoFile *rootf = (OstreeRepoFile*) root; + + mtree = ostree_mutable_tree_new_from_checksum (repo, + ostree_repo_file_tree_get_contents_checksum (rootf), + ostree_repo_file_tree_get_metadata_checksum (rootf)); + + if (opt_selinux_policy_from_base) + { + g_assert (modifier); + if (!ostree_repo_commit_modifier_set_sepolicy_from_commit (modifier, repo, base_commit, cancellable, error)) + goto out; + /* Don't try to handle it twice */ + opt_selinux_policy_from_base = FALSE; + } + } + else + { + mtree = ostree_mutable_tree_new (); + } + /* Convert implicit . or explicit path via argv into * --tree=dir= so that we only have one primary code path below. diff --git a/tests/archive-test.sh b/tests/archive-test.sh index 42b232bf..1e63a35b 100644 --- a/tests/archive-test.sh +++ b/tests/archive-test.sh @@ -68,3 +68,11 @@ echo "ok cat-file" cd ${test_tmpdir} $OSTREE fsck echo "ok fsck" + +mkdir -p test-overlays +date > test-overlays/overlaid-file +$OSTREE commit ${COMMIT_ARGS} -b test-base --base test2 --owner-uid 42 --owner-gid 42 test-overlays/ +$OSTREE ls -R test-base > ls.txt +assert_streq "$(wc -l < ls.txt)" 14 +assert_streq "$(grep '42.*42' ls.txt | wc -l)" 2 +echo "ok commit overlay base" diff --git a/tests/test-archivez.sh b/tests/test-archivez.sh index 0dccd737..c27ce03f 100755 --- a/tests/test-archivez.sh +++ b/tests/test-archivez.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo '1..12' +echo '1..13' setup_test_repository "archive" From ef1aeb7458612e933b0ee58ae0eb6c7590622b5d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Tue, 7 Apr 2020 01:11:29 +0300 Subject: [PATCH 096/177] lib/repo-pull: fix GPG check while pulling remote If GPG support is disabled in a build time we should to check if any of options "gpg_verify" or "gpg_verify_summary" is set to TRUE instead of checking if they are passed via options while pulling from remote. Fixed the failure with assertion of `ostree find-remotes --pull --mirror` calling (`tests/test-pull-collections.sh`) if libostree has been compiled without GPG support. Signed-off-by: Denis Pynkin --- src/libostree/ostree-repo-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 454eaf8a..56c0508d 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3856,7 +3856,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, #ifdef OSTREE_DISABLE_GPGME /* Explicitly fail here if gpg verification is requested and we have no GPG support */ - if (opt_gpg_verify_set || opt_gpg_verify_summary_set) + if (pull_data->gpg_verify || pull_data->gpg_verify_summary) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, "'%s': GPG feature is disabled in a build time", From 248596465c121bb50e25757e5cb5872ad3601060 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 7 Apr 2020 13:07:55 +0000 Subject: [PATCH 097/177] OWNERS: add d4s to reviewers He did a lot of work on signing and will continue that. --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index 08da5513..47a7bc81 100644 --- a/OWNERS +++ b/OWNERS @@ -16,3 +16,4 @@ reviewers: - lucab - mwleeds - rfairley + - d4s From b6040143e521d16fb2ea4e390232bae8ea5c26a6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 14 Apr 2020 22:18:13 +0000 Subject: [PATCH 098/177] Only enable "dummy" signature type with opt-in env variable I don't want to even have to think about people using this in production. --- src/libostree/ostree-sign-dummy.c | 16 +++++++++++++++- tests/test-signed-commit.sh | 13 ++++++++++++- tests/test-signed-pull-summary.sh | 3 +++ tests/test-signed-pull.sh | 2 ++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 722d461b..82575dc5 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -53,6 +53,14 @@ ostree_sign_dummy_iface_init (OstreeSignInterface *self); G_DEFINE_TYPE_WITH_CODE (OstreeSignDummy, _ostree_sign_dummy, G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE (OSTREE_TYPE_SIGN, ostree_sign_dummy_iface_init)); +static gboolean +check_dummy_sign_enabled (GError **error) +{ + if (g_strcmp0 (g_getenv ("OSTREE_DUMMY_SIGN_ENABLED"), "1") != 0) + return glnx_throw (error, "dummy signature type is only for ostree testing"); + return TRUE; +} + static void ostree_sign_dummy_iface_init (OstreeSignInterface *self) { @@ -83,6 +91,8 @@ _ostree_sign_dummy_init (OstreeSignDummy *self) gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **error) { + if (!check_dummy_sign_enabled (error)) + return FALSE; OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); @@ -95,7 +105,6 @@ gboolean ostree_sign_dummy_set_sk (OstreeSign *self, GVariant *key, GError **err gboolean ostree_sign_dummy_set_pk (OstreeSign *self, GVariant *key, GError **error) { - OstreeSignDummy *sign = _ostree_sign_dummy_get_instance_private(OSTREE_SIGN_DUMMY(self)); g_free(sign->pk_ascii); @@ -111,6 +120,8 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, GCancellable *cancellable, GError **error) { + if (!check_dummy_sign_enabled (error)) + return FALSE; g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -145,6 +156,9 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, GVariant *signatures, GError **error) { + if (!check_dummy_sign_enabled (error)) + return FALSE; + g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); g_return_val_if_fail (data != NULL, FALSE); diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 6730a6df..3ac3f3c1 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -23,7 +23,10 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..10" +echo "1..11" + +# This is explicitly opt in for testing +export OSTREE_DUMMY_SIGN_ENABLED=1 mkdir ${test_tmpdir}/repo ostree_repo_init repo --mode="archive" @@ -51,6 +54,14 @@ COMMIT="$(ostree --repo=${test_tmpdir}/repo rev-parse main)" ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} echo "ok commit with dummy signing" +if ${CMD_PREFIX} env -u OSTREE_DUMMY_SIGN_ENABLED ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} 2>err.txt; then + fatal "verified dummy signature without env" +fi +# FIXME the error message here is broken +#assert_file_has_content_literal err.txt 'dummy signature type is only for ostree testing' +assert_file_has_content_literal err.txt ' No valid signatures found' +echo "ok dummy sig requires env" + # tests below require libsodium support if ! has_libsodium; then echo "ok Detached ed25519 signature # SKIP due libsodium unavailability" diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh index ee731e86..c328d288 100755 --- a/tests/test-signed-pull-summary.sh +++ b/tests/test-signed-pull-summary.sh @@ -27,6 +27,9 @@ set -euo pipefail echo "1..14" +# This is explicitly opt in for testing +export OSTREE_DUMMY_SIGN_ENABLED=1 + repo_reinit () { ARGS="$*" cd ${test_tmpdir} diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 2c677d46..6d1afe29 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -25,6 +25,8 @@ set -euo pipefail echo "1..11" +# This is explicitly opt in for testing +export OSTREE_DUMMY_SIGN_ENABLED=1 setup_fake_remote_repo1 "archive" repo_mode="archive" From 452840f2833a8db08d77021d03c6f6f1ce7c98d0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 15 Apr 2020 12:36:52 -0400 Subject: [PATCH 099/177] ci: Adapt to use new fcosKola semantics This was changed recently and broke us since we do explicitly call `fcosKola` instead of implicitly via `fcosBuild`. Adapt to the new semantics. --- .cci.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index eaa976f5..faae52a9 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -77,7 +77,7 @@ parallel fcos: { coreos-assembler build """) } - fcosKola("${env.WORKSPACE}") + fcosKola(cosaDir: "${env.WORKSPACE}") } }, buildopts: { From e035230c6bd842ce0e73c5c82fe6c57d5a54b44a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 15 Apr 2020 14:46:08 +0000 Subject: [PATCH 100/177] lib/pull: Two cosmetic internal function renames I'm mainly doing this to sanity check the CI state right now. However, I also want to more cleanly/clearly distinguish the "sign" code from the "gpg" code. Rename one function to include `gpg`. For the other...I think what it's really doing is using the remote config, so change it to include `remote` in its name. --- src/libostree/ostree-repo-pull.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 64a48ce3..888fa1e7 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1440,7 +1440,7 @@ static_deltapart_fetch_on_complete (GObject *object, #ifndef OSTREE_DISABLE_GPGME static gboolean -process_verify_result (OtPullData *pull_data, +process_gpg_verify_result (OtPullData *pull_data, const char *checksum, OstreeGpgVerifyResult *result, GError **error) @@ -1563,7 +1563,7 @@ _load_public_keys (OstreeSign *sign, } static gboolean -_ostree_repo_sign_verify (OstreeRepo *repo, +_sign_verify_for_remote (OstreeRepo *repo, const gchar *remote_name, GBytes *signed_data, GVariant *metadata, @@ -1629,7 +1629,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, { if (pull_data->gpg_verify || pull_data->sign_verify) - /* Shouldn't happen, but see comment in process_verify_result() */ + /* Shouldn't happen, but see comment in process_gpg_verify_result() */ if (g_hash_table_contains (pull_data->verified_commits, checksum)) return TRUE; @@ -1650,7 +1650,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, detached_metadata, keyring_remote, NULL, NULL, cancellable, error); - if (!process_verify_result (pull_data, checksum, result, error)) + if (!process_gpg_verify_result (pull_data, checksum, result, error)) return FALSE; } #endif /* OSTREE_DISABLE_GPGME */ @@ -1661,7 +1661,7 @@ ostree_verify_unwritten_commit (OtPullData *pull_data, if (detached_metadata == NULL) return glnx_throw (error, "Can't verify commit without detached metadata"); - if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) + if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) return glnx_prefix_error (error, "Can't verify commit"); /* Mark the commit as verified to avoid double verification @@ -1993,7 +1993,7 @@ scan_commit_object (OtPullData *pull_data, keyring_remote, cancellable, error); - if (!process_verify_result (pull_data, checksum, result, error)) + if (!process_gpg_verify_result (pull_data, checksum, result, error)) return FALSE; } #endif /* OSTREE_DISABLE_GPGME */ @@ -4411,7 +4411,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, bytes_sig, FALSE); - if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, &temp_error)) + if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, &temp_error)) { if (summary_from_cache) { @@ -4440,7 +4440,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (!_ostree_repo_sign_verify (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) + if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) goto out; } else @@ -6547,7 +6547,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, signatures, FALSE); - if (!_ostree_repo_sign_verify (self, name, summary, sig_variant, error)) + if (!_sign_verify_for_remote (self, name, summary, sig_variant, error)) goto out; } } From 8e7aea4473375a12ad479d910a1e20fabe23ed18 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 12 Apr 2020 18:04:06 +0000 Subject: [PATCH 101/177] Change signature opts to include type, cleanup error handling Previously we would pass the `verification-key` and `verification-file` to all backends, ignoring errors from loading keys until we found one that worked. Instead, change the options to be `verification--key` and `verification--file`, and then rework this to use standard error handling; barf explicitly if we can't load the public keys for example. Preserve the semantics of accepting the first valid signature. The first signature error is captured, the others are currently compressed into a `(and %d more)` prefix. And now that I look at this more closely there's a lot of duplication between the two code paths in pull.c for verifying; will dedup this next. --- src/libostree/ostree-repo-pull.c | 134 +++++++++++++++++----------- src/libostree/ostree-sign-ed25519.c | 2 +- tests/test-local-pull.sh | 12 +-- tests/test-signed-pull-summary.sh | 16 ++-- tests/test-signed-pull.sh | 16 ++-- 5 files changed, 106 insertions(+), 74 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 888fa1e7..263f1cfc 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1470,11 +1470,23 @@ process_gpg_verify_result (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ +static gboolean +get_signapi_remote_option (OstreeRepo *repo, + OstreeSign *sign, + const char *remote_name, + const char *keysuffix, + char **out_value, + GError **error) +{ + g_autofree char *key = g_strdup_printf ("verification-%s-%s", ostree_sign_get_name (sign), keysuffix); + return ostree_repo_get_remote_option (repo, remote_name, key, NULL, out_value, error); +} + /* _load_public_keys: * * Load public keys according remote's configuration: - * inlined key passed via config option `verification-key` or - * file name with public keys via `verification-file` option. + * inlined key passed via config option `verification--key` or + * file name with public keys via `verification--file` option. * * If both options are set then load all all public keys * both from file and inlined in config. @@ -1493,15 +1505,10 @@ _load_public_keys (OstreeSign *sign, gboolean loaded_from_file = TRUE; gboolean loaded_inlined = TRUE; - ostree_repo_get_remote_option (repo, - remote_name, - "verification-file", NULL, - &pk_file, NULL); - - ostree_repo_get_remote_option (repo, - remote_name, - "verification-key", NULL, - &pk_ascii, NULL); + if (!get_signapi_remote_option (repo, sign, remote_name, "file", &pk_file, error)) + return FALSE; + if (!get_signapi_remote_option (repo, sign, remote_name, "key", &pk_ascii, error)) + return FALSE; /* return TRUE if there is no configuration for remote */ if ((pk_file == NULL) &&(pk_ascii == NULL)) @@ -1571,9 +1578,10 @@ _sign_verify_for_remote (OstreeRepo *repo, { /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); - g_autoptr (GError) verification_error = NULL; - - glnx_throw (&verification_error, "signed with unknown key"); + guint n_invalid_signatures = 0; + guint n_unknown_signatures = 0; + g_autoptr (GError) last_sig_error = NULL; + gboolean found_sig = FALSE; for (char **iter=names; iter && *iter; iter++) { @@ -1581,10 +1589,12 @@ _sign_verify_for_remote (OstreeRepo *repo, g_autoptr (GVariant) signatures = NULL; const gchar *signature_key = NULL; GVariantType *signature_format = NULL; - g_autoptr (GError) local_error = NULL; - if ((sign = ostree_sign_get_by_name (*iter, &local_error)) == NULL) - continue; + if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) + { + n_unknown_signatures++; + continue; + } signature_key = ostree_sign_metadata_key (sign); signature_format = (GVariantType *) ostree_sign_metadata_format (sign); @@ -1598,24 +1608,37 @@ _sign_verify_for_remote (OstreeRepo *repo, continue; /* Try to load public key(s) according remote's configuration */ - if (_load_public_keys (sign, repo, remote_name, &local_error)) - { - /* Return true if any signature fit to pre-loaded public keys. - * If no keys configured -- then system configuration will be used */ - if (ostree_sign_data_verify (sign, - signed_data, - signatures, - &local_error)) - return TRUE; - } + if (!_load_public_keys (sign, repo, remote_name, error)) + return FALSE; - /* Save error message for better reason detection later if needed */ - glnx_prefix_error (&verification_error, "%s", local_error->message); + found_sig = TRUE; + + /* Return true if any signature fit to pre-loaded public keys. + * If no keys configured -- then system configuration will be used */ + if (!ostree_sign_data_verify (sign, + signed_data, + signatures, + last_sig_error ? NULL : &last_sig_error)) + { + n_invalid_signatures++; + continue; + } + /* Accept the first valid signature */ + return TRUE; } - /* In case if there were no signatures of known type - * or metadata contains invalid data */ - return glnx_throw (error, "%s", verification_error->message); + if (!found_sig) + { + if (n_unknown_signatures > 0) + return glnx_throw (error, "No signatures found (%d unknown type)", n_unknown_signatures); + return glnx_throw (error, "No signatures found"); + } + + g_assert (last_sig_error); + g_propagate_error (error, g_steal_pointer (&last_sig_error)); + if (n_invalid_signatures > 1) + glnx_prefix_error (error, "(%d other invalid signatures)", n_invalid_signatures-1); + return FALSE; } static gboolean @@ -2001,39 +2024,46 @@ scan_commit_object (OtPullData *pull_data, if (pull_data->sign_verify && !g_hash_table_contains (pull_data->verified_commits, checksum)) { - gboolean ret = FALSE; - g_autoptr (GError) verification_error = NULL; + g_autoptr(GError) last_verification_error = NULL; + gboolean found_any_signature = FALSE; + gboolean found_valid_signature = FALSE; /* list all signature types in detached metadata and check if signed by any? */ g_auto (GStrv) names = ostree_sign_list_names(); - for (char **iter=names; !ret && iter && *iter; iter++) + for (char **iter=names; iter && *iter; iter++) { g_autoptr (OstreeSign) sign = NULL; - g_autoptr (GError) local_error = NULL; - if ((sign = ostree_sign_get_by_name (*iter, &local_error)) == NULL) + if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) continue; /* Try to load public key(s) according remote's configuration */ - if (_load_public_keys (sign, pull_data->repo, pull_data->remote_name, &local_error)) + if (!_load_public_keys (sign, pull_data->repo, pull_data->remote_name, error)) + return FALSE; + + found_any_signature = TRUE; + + /* Set return to true if any sign fit */ + if (ostree_sign_commit_verify (sign, + pull_data->repo, + checksum, + cancellable, + last_verification_error ? NULL : &last_verification_error)) { - - /* Set return to true if any sign fit */ - if (ostree_sign_commit_verify (sign, - pull_data->repo, - checksum, - cancellable, - &local_error)) - ret = TRUE; + found_valid_signature = TRUE; + break; } - - /* Save error message for better reason detection later if needed */ - if (!ret) - glnx_prefix_error (&verification_error, "%s", local_error->message); } - if (!ret) - return glnx_throw (error, "Can't verify commit %s: %s", checksum, verification_error->message); + if (!found_any_signature) + return glnx_throw (error, "No signatures found for commit %s", checksum); + + if (!found_valid_signature) + { + g_assert (last_verification_error); + g_propagate_error (error, g_steal_pointer (&last_verification_error)); + return glnx_prefix_error (error, "Can't verify commit %s", checksum); + } } /* If we found a legacy transaction flag, assume we have to scan. diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 8df61aed..4d984d1e 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -242,7 +242,7 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, } } - return glnx_throw (error, "Not able to verify: no valid signatures found"); + return glnx_throw (error, "no valid ed25519 signatures found"); #endif /* HAVE_LIBSODIUM */ return FALSE; diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index 2b7ca13a..8cbc9732 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -121,27 +121,29 @@ if has_libsodium; then mkdir repo8 ostree_repo_init repo8 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo8 remote add --set=verification-key="${ED25519PUBLIC}" origin repo + ${CMD_PREFIX} ostree --repo=repo8 remote add --set=verification-ed25519-key="${ED25519PUBLIC}" origin repo cat repo8/config - if ${CMD_PREFIX} ostree --repo=repo8 pull-local --remote=origin --sign-verify repo test2 2>&1; then + if ${CMD_PREFIX} ostree --repo=repo8 pull-local --remote=origin --sign-verify repo test2 2>err.txt; then assert_not_reached "Ed25519 signature verification unexpectedly succeeded" fi + assert_file_has_content err.txt 'ed25519: commit have no signatures of my type' echo "ok --sign-verify with no signature" ${OSTREE} sign test2 ${ED25519SECRET} mkdir repo9 ostree_repo_init repo9 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo9 remote add --set=verification-key="$(gen_ed25519_random_public)" origin repo - if ${CMD_PREFIX} ostree --repo=repo9 pull-local --remote=origin --sign-verify repo test2 2>&1; then + ${CMD_PREFIX} ostree --repo=repo9 remote add --set=verification-ed25519-key="$(gen_ed25519_random_public)" origin repo + if ${CMD_PREFIX} ostree --repo=repo9 pull-local --remote=origin --sign-verify repo test2 2>err.txt; then assert_not_reached "Ed25519 signature verification unexpectedly succeeded" fi + assert_file_has_content err.txt 'no valid ed25519 signatures found' echo "ok --sign-verify with wrong signature" mkdir repo10 ostree_repo_init repo10 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo10 remote add --set=verification-key="${ED25519PUBLIC}" origin repo + ${CMD_PREFIX} ostree --repo=repo10 remote add --set=verification-ed25519-key="${ED25519PUBLIC}" origin repo ${CMD_PREFIX} ostree --repo=repo10 pull-local --remote=origin --sign-verify repo test2 echo "ok --sign-verify" else diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh index c328d288..7a7dd073 100755 --- a/tests/test-signed-pull-summary.sh +++ b/tests/test-signed-pull-summary.sh @@ -115,7 +115,7 @@ do ${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} cd ${test_tmpdir} - repo_reinit --set=verification-key=${PUBLIC_KEY} + repo_reinit --set=verification-${engine}-key=${PUBLIC_KEY} ${OSTREE} --repo=repo pull origin main assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig @@ -136,7 +136,7 @@ do echo "ok ${engine} prune summary cache" cd ${test_tmpdir} - repo_reinit --set=verification-key=${PUBLIC_KEY} + repo_reinit --set=verification-${engine}-key=${PUBLIC_KEY} mkdir cachedir ${OSTREE} --repo=repo pull --cache-dir=cachedir origin main assert_not_has_file repo/tmp/cache/summaries/origin @@ -152,13 +152,13 @@ do echo "ok ${engine} pull with signed summary and cachedir" cd ${test_tmpdir} - repo_reinit --set=verification-key=${PUBLIC_KEY} + repo_reinit --set=verification-${engine}-key=${PUBLIC_KEY} mv ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{,.good} echo invalid > ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with invalid ${engine} signature" fi - assert_file_has_content err.txt "signed with unknown key" + assert_file_has_content err.txt "No signatures found" mv ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.good,} echo "ok ${engine} pull with invalid ${engine} summary signature fails" @@ -167,7 +167,7 @@ do ${OSTREE} --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u ${COMMIT_SIGN} cd ${test_tmpdir} - repo_reinit --set=verification-key=${PUBLIC_KEY} + repo_reinit --set=verification-${engine}-key=${PUBLIC_KEY} ${OSTREE} --repo=repo pull origin main echo "ok ${engine} pull delta with signed summary" @@ -212,7 +212,7 @@ cd ${test_tmpdir} # Reset to the old valid summary and pull to cache it cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.1,} cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.1,} -repo_reinit --set=verification-key=${PUBLIC_KEY} +repo_reinit --set=verification-ed25519-key=${PUBLIC_KEY} ${OSTREE} --repo=repo pull origin main assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig @@ -226,7 +226,7 @@ cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with old summary" fi -assert_file_has_content err.txt "signed with unknown key" +assert_file_has_content err.txt "no valid ed25519 signatures found" assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 @@ -246,7 +246,7 @@ echo "ok ${engine} pull with signed summary remote old summary" # Reset to the old valid summary and pull to cache it cp ${test_tmpdir}/ostree-srv/gnomerepo/summary{.1,} cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.1,} -repo_reinit --set=verification-key=${PUBLIC_KEY} +repo_reinit --set=verification-ed25519-key=${PUBLIC_KEY} ${OSTREE} --repo=repo pull origin main assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 6d1afe29..a8d52bc5 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -80,22 +80,22 @@ if ${CMD_PREFIX} ostree --repo=repo pull origin main; then fi echo "ok pull failure without keys preloaded" -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "somewrongkey" +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy-key "somewrongkey" if ${CMD_PREFIX} ostree --repo=repo pull origin main; then assert_not_reached "pull with unknown key unexpectedly succeeded" fi echo "ok pull failure with incorrect key option" -${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-key -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "/non/existing/file" +${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-dummy-key +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy-file "/non/existing/file" if ${CMD_PREFIX} ostree --repo=repo pull origin main; then assert_not_reached "pull with unknown keys file unexpectedly succeeded" fi echo "ok pull failure with incorrect keys file option" # Test with correct dummy key -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${DUMMYSIGN}" -${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-file +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy-key "${DUMMYSIGN}" +${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-dummy-file test_signed_pull "dummy" "" if ! has_libsodium; then @@ -117,7 +117,7 @@ SECRET=${ED25519SECRET} COMMIT_ARGS="--sign=${SECRET} --sign-type=ed25519" repo_init --set=sign-verify=true -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-key "${PUBLIC}" +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-ed25519-key "${PUBLIC}" test_signed_pull "ed25519" "key" # Prepare files with public ed25519 signatures @@ -130,13 +130,13 @@ for((i=0;i<100;i++)); do done > ${PUBKEYS} # Test case with the file containing incorrect signatures and with the correct key set -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "${PUBKEYS}" +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-ed25519-file "${PUBKEYS}" test_signed_pull "ed25519" "key+file" # Add correct key into the list echo ${PUBLIC} >> ${PUBKEYS} repo_init --set=sign-verify=true -${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-file "${PUBKEYS}" +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-ed25519-file "${PUBKEYS}" test_signed_pull "ed25519" "file" From a6efae909f24a9555d032397d49135550aa19556 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Apr 2020 14:15:04 +0000 Subject: [PATCH 102/177] ci: Build minimal without libsodium too The goal is to test "no options" build - and eventually tests. (We're not actually including libsodium in the cosa buildroot right now, but we may in the future) --- .cci.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index faae52a9..484c3b14 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -41,7 +41,7 @@ minimal: { ./configure --without-curl --without-soup --disable-gtk-doc --disable-man \ --disable-rust --without-libarchive --without-selinux --without-smack \ --without-openssl --without-avahi --without-libmount --disable-rofiles-fuse \ - --disable-experimental-api + --disable-experimental-api --without-libsodium make """) } From d1e5ed379ebcfca7d9623c006528475bac88993b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Apr 2020 16:43:43 +0000 Subject: [PATCH 103/177] Use `sign-ed25519` for the feature name `libsodium` is an implementation detail. In particular, I'd like to consider using OpenSSL for ed25519 (if libsodium isn't configured and openssl is). So switch the name of the exposed feature and adjust the tests. --- configure.ac | 2 +- tests/libtest.sh | 4 ++-- tests/test-local-pull.sh | 2 +- tests/test-signed-commit.sh | 2 +- tests/test-signed-pull-summary.sh | 4 ++-- tests/test-signed-pull.sh | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 5d9d2a23..92e47b2a 100644 --- a/configure.ac +++ b/configure.ac @@ -253,7 +253,7 @@ AS_IF([test x$with_libsodium != xno], [ AS_IF([ test x$have_libsodium = xno ], [ AC_MSG_ERROR([Need LIBSODIUM version $LIBSODIUM_DEPENDENCY or later]) ]) - OSTREE_FEATURES="$OSTREE_FEATURES libsodium" + OSTREE_FEATURES="$OSTREE_FEATURES sign-ed25519" ], with_libsodium=no ) AM_CONDITIONAL(USE_LIBSODIUM, test "x$have_libsodium" = xyes) diff --git a/tests/libtest.sh b/tests/libtest.sh index c473fd82..315c4df5 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -679,10 +679,10 @@ libtest_cleanup_gpg () { } libtest_exit_cmds+=(libtest_cleanup_gpg) -has_libsodium () { +has_sign_ed25519 () { local ret ${CMD_PREFIX} ostree --version > version.txt - grep -q -e '- libsodium' version.txt + grep -q -e '- sign-ed25519' version.txt ret=$? rm -f version.txt return ${ret} diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index 8cbc9732..d443421a 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -116,7 +116,7 @@ for src_object in `find repo/objects -name '*.filez'`; do done echo "ok pull-local z2 to z2 default hardlink" -if has_libsodium; then +if has_sign_ed25519; then gen_ed25519_keys mkdir repo8 diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 3ac3f3c1..dd76f28f 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -63,7 +63,7 @@ assert_file_has_content_literal err.txt ' No valid signatures found' echo "ok dummy sig requires env" # tests below require libsodium support -if ! has_libsodium; then +if ! has_sign_ed25519; then echo "ok Detached ed25519 signature # SKIP due libsodium unavailability" echo "ok ed25519 signature verified # SKIP due libsodium unavailability" echo "ok multiple signing # SKIP due libsodium unavailability" diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh index 7a7dd073..6a240635 100755 --- a/tests/test-signed-pull-summary.sh +++ b/tests/test-signed-pull-summary.sh @@ -51,7 +51,7 @@ do PUBLIC_KEY="dummysign" ;; ed25519) - if ! has_libsodium; then + if ! has_sign_ed25519; then echo "ok ${engine} pull mirror summary # SKIP due libsodium unavailability" echo "ok ${engine} pull with signed summary # SKIP due libsodium unavailability" echo "ok ${engine} prune summary cache # SKIP due libsodium unavailability" @@ -173,7 +173,7 @@ do done -if ! has_libsodium; then +if ! has_sign_ed25519; then echo "ok ${engine} pull with signed summary remote old summary # SKIP due libsodium unavailability" echo "ok ${engine} pull with signed summary broken cache # SKIP due libsodium unavailability" exit 0 diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index a8d52bc5..075c5f2b 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -98,7 +98,7 @@ ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy ${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-dummy-file test_signed_pull "dummy" "" -if ! has_libsodium; then +if ! has_sign_ed25519; then echo "ok ed25519-key pull signed commit # SKIP due libsodium unavailability" echo "ok ed25519-key re-pull signature for stored commit # SKIP due libsodium unavailability" echo "ok ed25519-key+file pull signed commit # SKIP due libsodium unavailability" From e63647c6d05f4ff4402f3521aae9e82393178b88 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 15 Apr 2020 22:16:58 +0000 Subject: [PATCH 104/177] travis: Add some libsodium coverage As far as I can tell we're not gating on this right now. From a quick glance, it looks like Debian stable has `libsodium-dev` but only Ubuntu eoan does which we're not testing right now. --- .travis.yml | 5 +++-- ci/travis-Dockerfile.in | 2 +- ci/travis-install.sh | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a77bf827..227ad4b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,9 @@ dist: trusty sudo: required env: - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl" + # debian has libsodium-dev, ubuntu doesn't in core at least + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-libsodium" ci_pkgs="libsodium-dev" + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl --with-libsodium --without-gpgme" ci_pkgs="libsodium-dev" - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial - ci_docker=ubuntu:bionic ci_distro=ubuntu ci_suite=bionic diff --git a/ci/travis-Dockerfile.in b/ci/travis-Dockerfile.in index df999e78..9e47876e 100644 --- a/ci/travis-Dockerfile.in +++ b/ci/travis-Dockerfile.in @@ -2,7 +2,7 @@ FROM @ci_docker@ ENV container docker ADD ci/travis-install.sh /travis-install.sh -RUN ci_suite="@ci_suite@" ci_distro="@ci_distro@" ci_in_docker=yes /travis-install.sh +RUN ci_suite="@ci_suite@" ci_distro="@ci_distro@" ci_pkgs="@ci_pkgs@" ci_in_docker=yes /travis-install.sh ADD . /home/user/ostree RUN chown -R user:user /home/user/ostree diff --git a/ci/travis-install.sh b/ci/travis-install.sh index 8992aed0..b455fd3e 100755 --- a/ci/travis-install.sh +++ b/ci/travis-install.sh @@ -64,6 +64,7 @@ if [ -n "$ci_docker" ]; then -e "s/@ci_distro@/${ci_distro}/" \ -e "s/@ci_docker@/${ci_docker}/" \ -e "s/@ci_suite@/${ci_suite}/" \ + -e "s/@ci_pkgs@/${ci_pkgs}/" \ < ci/travis-Dockerfile.in > Dockerfile exec docker build -t ci-image . fi @@ -111,6 +112,7 @@ case "$ci_distro" in procps \ zlib1g-dev \ python3-yaml \ + ${ci_pkgs:-} \ ${NULL} if [ "$ci_in_docker" = yes ]; then From 735b03dbf99ffeef678e9c1e2fe7f757ccaafc51 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 16 Apr 2020 21:57:51 +0300 Subject: [PATCH 105/177] Add ci_pkgs to travis-install.sh Signed-off-by: Denis Pynkin --- ci/travis-install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/travis-install.sh b/ci/travis-install.sh index b455fd3e..c28a4111 100755 --- a/ci/travis-install.sh +++ b/ci/travis-install.sh @@ -53,6 +53,9 @@ NULL= # ci_configopts: Additional arguments for configure : "${ci_configopts:=}" +# ci_pkgs: Additional packages to be installed +: "${ci_pkgs:=}" + if [ $(id -u) = 0 ]; then sudo= else From 0807f100e2bf1b578f2752825d5415ec288494e6 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 16 Apr 2020 22:12:51 +0300 Subject: [PATCH 106/177] Fix the lost line separator Signed-off-by: Denis Pynkin --- ci/travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis-build.sh b/ci/travis-build.sh index 3fd969bd..942c0e77 100755 --- a/ci/travis-build.sh +++ b/ci/travis-build.sh @@ -85,7 +85,7 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ../configure \ --enable-always-build-tests \ - ${ci_configopts} + ${ci_configopts} \ "$@" ${make} From a17cdec936c0421329efe3d99027eb2161b53cc4 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Thu, 16 Apr 2020 22:29:55 +0300 Subject: [PATCH 107/177] Add the same config options for distcheck Signed-off-by: Denis Pynkin --- ci/travis-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis-build.sh b/ci/travis-build.sh index 942c0e77..e31da81f 100755 --- a/ci/travis-build.sh +++ b/ci/travis-build.sh @@ -91,7 +91,7 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ${make} [ "$ci_test" = no ] || ${make} check || maybe_fail_tests cat test-suite.log || : -[ "$ci_test" = no ] || ${make} distcheck || maybe_fail_tests +[ "$ci_test" = no ] || ${make} distcheck DISTCHECK_CONFIGURE_FLAGS="${ci_configopts}" || maybe_fail_tests cat test-suite.log || : ${make} install DESTDIR=$(pwd)/DESTDIR From b7f48a74c0f97237de0576b2afe2ffb009f8d448 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Apr 2020 01:16:51 +0000 Subject: [PATCH 108/177] lib: Move internal binding verification API to repo.c `ostree-repo-pull.c` is rather monstrous; I plan to split it up a bit. There's actually already a `pull-private.h` but that's just for the binding verification API. I think that one isn't really pull specific. Let's move it into the "catchall" `repo.c`. --- src/libostree/ostree-cmdprivate.c | 1 - src/libostree/ostree-repo-private.h | 6 ++ src/libostree/ostree-repo-pull-private.h | 5 -- src/libostree/ostree-repo-pull.c | 104 ----------------------- src/libostree/ostree-repo.c | 104 +++++++++++++++++++++++ 5 files changed, 110 insertions(+), 110 deletions(-) diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index de82521c..86793790 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -24,7 +24,6 @@ #include "ostree-cmdprivate.h" #include "ostree-repo-private.h" #include "ostree-core-private.h" -#include "ostree-repo-pull-private.h" #include "ostree-repo-static-delta-private.h" #include "ostree-sysroot-private.h" #include "ostree-bootloader-grub2.h" diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 9f722a39..e52f9f0b 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -502,4 +502,10 @@ _ostree_tmpf_fsverity (OstreeRepo *self, GLnxTmpfile *tmpf, GError **error); +gboolean +_ostree_repo_verify_bindings (const char *collection_id, + const char *ref_name, + GVariant *commit, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index f50697b9..918c4617 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -25,10 +25,5 @@ G_BEGIN_DECLS -gboolean -_ostree_repo_verify_bindings (const char *collection_id, - const char *ref_name, - GVariant *commit, - GError **error); G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 263f1cfc..f376da82 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -29,7 +29,6 @@ #include "libglnx.h" #include "ostree.h" #include "otutil.h" -#include "ostree-repo-pull-private.h" #include "ostree-repo-private.h" #ifdef HAVE_LIBCURL_OR_LIBSOUP @@ -1705,109 +1704,6 @@ commitstate_is_partial (OtPullData *pull_data, #endif /* HAVE_LIBCURL_OR_LIBSOUP */ -/** - * _ostree_repo_verify_bindings: - * @collection_id: (nullable): Locally specified collection ID for the remote - * the @commit was retrieved from, or %NULL if none is configured - * @ref_name: (nullable): Ref name the commit was retrieved using, or %NULL if - * the commit was retrieved by checksum - * @commit: Commit data to check - * @error: Return location for a #GError, or %NULL - * - * Verify the ref and collection bindings. - * - * The ref binding is verified only if it exists. But if we have the - * collection ID specified in the remote configuration (@collection_id is - * non-%NULL) then the ref binding must exist, otherwise the verification will - * fail. Parts of the verification can be skipped by passing %NULL to the - * @ref_name parameter (in case we requested a checksum directly, without - * looking it up from a ref). - * - * The collection binding is verified only when we have collection ID - * specified in the remote configuration. If it is specified, then the - * binding must exist and must be equal to the remote repository - * collection ID. - * - * Returns: %TRUE if bindings are correct, %FALSE otherwise - * Since: 2017.14 - */ -gboolean -_ostree_repo_verify_bindings (const char *collection_id, - const char *ref_name, - GVariant *commit, - GError **error) -{ - g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0); - g_autofree const char **refs = NULL; - if (!g_variant_lookup (metadata, - OSTREE_COMMIT_META_KEY_REF_BINDING, - "^a&s", - &refs)) - { - /* Early return here - if the remote collection ID is NULL, then - * we certainly will not verify the collection binding in the - * commit. - */ - if (collection_id == NULL) - return TRUE; - - return glnx_throw (error, - "Expected commit metadata to have ref " - "binding information, found none"); - } - - if (ref_name != NULL) - { - if (!g_strv_contains ((const char *const *) refs, ref_name)) - { - g_autoptr(GString) refs_dump = g_string_new (NULL); - const char *refs_str; - - if (refs != NULL && (*refs) != NULL) - { - for (const char **iter = refs; *iter != NULL; ++iter) - { - const char *ref = *iter; - - if (refs_dump->len > 0) - g_string_append (refs_dump, ", "); - g_string_append_printf (refs_dump, "‘%s’", ref); - } - - refs_str = refs_dump->str; - } - else - { - refs_str = "no refs"; - } - - return glnx_throw (error, "Commit has no requested ref ‘%s’ " - "in ref binding metadata (%s)", - ref_name, refs_str); - } - } - - if (collection_id != NULL) - { - const char *collection_id_binding; - if (!g_variant_lookup (metadata, - OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, - "&s", - &collection_id_binding)) - return glnx_throw (error, - "Expected commit metadata to have collection ID " - "binding information, found none"); - if (!g_str_equal (collection_id_binding, collection_id)) - return glnx_throw (error, - "Commit has collection ID ‘%s’ in collection binding " - "metadata, while the remote it came from has " - "collection ID ‘%s’", - collection_id_binding, collection_id); - } - - return TRUE; -} - /* Reads the collection-id of a given remote from the repo * configuration. */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f665106d..97ce95b8 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -6241,3 +6241,107 @@ ostree_repo_get_bootloader (OstreeRepo *self) return self->bootloader; } + + +/** + * _ostree_repo_verify_bindings: + * @collection_id: (nullable): Locally specified collection ID for the remote + * the @commit was retrieved from, or %NULL if none is configured + * @ref_name: (nullable): Ref name the commit was retrieved using, or %NULL if + * the commit was retrieved by checksum + * @commit: Commit data to check + * @error: Return location for a #GError, or %NULL + * + * Verify the ref and collection bindings. + * + * The ref binding is verified only if it exists. But if we have the + * collection ID specified in the remote configuration (@collection_id is + * non-%NULL) then the ref binding must exist, otherwise the verification will + * fail. Parts of the verification can be skipped by passing %NULL to the + * @ref_name parameter (in case we requested a checksum directly, without + * looking it up from a ref). + * + * The collection binding is verified only when we have collection ID + * specified in the remote configuration. If it is specified, then the + * binding must exist and must be equal to the remote repository + * collection ID. + * + * Returns: %TRUE if bindings are correct, %FALSE otherwise + * Since: 2017.14 + */ +gboolean +_ostree_repo_verify_bindings (const char *collection_id, + const char *ref_name, + GVariant *commit, + GError **error) +{ + g_autoptr(GVariant) metadata = g_variant_get_child_value (commit, 0); + g_autofree const char **refs = NULL; + if (!g_variant_lookup (metadata, + OSTREE_COMMIT_META_KEY_REF_BINDING, + "^a&s", + &refs)) + { + /* Early return here - if the remote collection ID is NULL, then + * we certainly will not verify the collection binding in the + * commit. + */ + if (collection_id == NULL) + return TRUE; + + return glnx_throw (error, + "Expected commit metadata to have ref " + "binding information, found none"); + } + + if (ref_name != NULL) + { + if (!g_strv_contains ((const char *const *) refs, ref_name)) + { + g_autoptr(GString) refs_dump = g_string_new (NULL); + const char *refs_str; + + if (refs != NULL && (*refs) != NULL) + { + for (const char **iter = refs; *iter != NULL; ++iter) + { + const char *ref = *iter; + + if (refs_dump->len > 0) + g_string_append (refs_dump, ", "); + g_string_append_printf (refs_dump, "‘%s’", ref); + } + + refs_str = refs_dump->str; + } + else + { + refs_str = "no refs"; + } + + return glnx_throw (error, "Commit has no requested ref ‘%s’ " + "in ref binding metadata (%s)", + ref_name, refs_str); + } + } + + if (collection_id != NULL) + { + const char *collection_id_binding; + if (!g_variant_lookup (metadata, + OSTREE_COMMIT_META_KEY_COLLECTION_BINDING, + "&s", + &collection_id_binding)) + return glnx_throw (error, + "Expected commit metadata to have collection ID " + "binding information, found none"); + if (!g_str_equal (collection_id_binding, collection_id)) + return glnx_throw (error, + "Commit has collection ID ‘%s’ in collection binding " + "metadata, while the remote it came from has " + "collection ID ‘%s’", + collection_id_binding, collection_id); + } + + return TRUE; +} From 40a2fb3b076a5e6e5076c9d13359d79cc6adb467 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 17 Apr 2020 16:46:49 +0300 Subject: [PATCH 109/177] tests/signed-commit: fix the test of well-known places Commit e474033e removed the redirection of incorrect public keys aimed to generate a lot of files without correct public signature. Fix the test by returning back the creation of files containing incorrect public keys for ed25519. Signed-off-by: Denis Pynkin --- tests/test-signed-commit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index dd76f28f..4dcf38a4 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -182,7 +182,7 @@ echo "ok sign with ed25519 keys file" mkdir -p ${test_tmpdir}/{trusted,revoked}.ed25519.d for((i=0;i<100;i++)); do # Generate some key files with random public signatures - gen_ed25519_random_public + gen_ed25519_random_public > ${test_tmpdir}/trusted.ed25519.d/signature_$i done # Check no valid public keys are available if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-dir=${test_tmpdir} ${COMMIT}; then From 6aeca233f6139aeadc5d6852ec386c54079ed019 Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 17 Apr 2020 19:22:37 +0300 Subject: [PATCH 110/177] sign: rename option for enabling ed25519 Use option `--with-ed25519-libsodium` instead of `--with-libsodium` to enable ed25519 signature engine. This allows to use later different implementations of ed25519 signing/verification. For instance, based on openssl. Signed-off-by: Denis Pynkin --- .travis.yml | 4 ++-- configure.ac | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 227ad4b3..92217682 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ sudo: required env: # debian has libsodium-dev, ubuntu doesn't in core at least - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-libsodium" ci_pkgs="libsodium-dev" - - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl --with-libsodium --without-gpgme" ci_pkgs="libsodium-dev" + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-ed25519-libsodium" ci_pkgs="libsodium-dev" + - ci_docker=debian:buster-slim ci_distro=debian ci_suite=stretch ci_configopts="--with-curl --with-ed25519-libsodium --without-gpgme" ci_pkgs="libsodium-dev" - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial - ci_docker=ubuntu:bionic ci_distro=ubuntu ci_suite=bionic diff --git a/configure.ac b/configure.ac index 92e47b2a..f39a5006 100644 --- a/configure.ac +++ b/configure.ac @@ -244,17 +244,17 @@ AM_CONDITIONAL(USE_GPGME, test "x$have_gpgme" = xyes) LIBSODIUM_DEPENDENCY="1.0.14" -AC_ARG_WITH(libsodium, - AS_HELP_STRING([--with-libsodium], [Use libsodium @<:@default=no@:>@]), - [], [with_libsodium=no]) -AS_IF([test x$with_libsodium != xno], [ +AC_ARG_WITH(ed25519_libsodium, + AS_HELP_STRING([--with-ed25519-libsodium], [Use libsodium for ed25519 @<:@default=no@:>@]), + [], [with_ed25519_libsodium=no]) +AS_IF([test x$with_ed25519_libsodium != xno], [ AC_DEFINE([HAVE_LIBSODIUM], 1, [Define if using libsodium]) PKG_CHECK_MODULES(OT_DEP_LIBSODIUM, libsodium >= $LIBSODIUM_DEPENDENCY, have_libsodium=yes, have_libsodium=no) AS_IF([ test x$have_libsodium = xno ], [ AC_MSG_ERROR([Need LIBSODIUM version $LIBSODIUM_DEPENDENCY or later]) ]) OSTREE_FEATURES="$OSTREE_FEATURES sign-ed25519" -], with_libsodium=no ) +], with_ed25519_libsodium=no ) AM_CONDITIONAL(USE_LIBSODIUM, test "x$have_libsodium" = xyes) LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" @@ -641,7 +641,7 @@ echo " cryptographic checksums: $with_crypto systemd: $with_libsystemd libmount: $with_libmount - libsodium (ed25519 signatures): $with_libsodium + libsodium (ed25519 signatures): $with_ed25519_libsodium libarchive (parse tar files directly): $with_libarchive static deltas: yes (always enabled now) O_TMPFILE: $enable_otmpfile From bb8fd5a2c4b91d8fd41e831887f053bb63b13ac5 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 17 Apr 2020 14:18:28 -0400 Subject: [PATCH 111/177] lib/commit: Add more error prefixing We think we're hitting an error in that function in the Fedora infra. Add some more error prefixing to help debugging. --- src/libostree/ostree-repo-commit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 3f2f2beb..6712b53e 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1943,6 +1943,9 @@ cleanup_txn_dir (OstreeRepo *self, GCancellable *cancellable, GError **error) { + const char *errprefix = glnx_strjoina ("Cleaning up txn dir ", path); + GLNX_AUTO_PREFIX_ERROR (errprefix, error); + g_auto(GLnxLockFile) lockfile = { 0, }; gboolean did_lock; From df065ad766c325bac2fa9a83ea1af7e1616abe16 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 17 Apr 2020 14:20:25 -0400 Subject: [PATCH 112/177] lib: Rename function for staging dir check Rename the function to more accurately reflect what it does, which is to check whether the filename has the `staging-` prefix. --- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo-private.h | 2 +- src/libostree/ostree-repo.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 6712b53e..8c4bd3fe 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2011,7 +2011,7 @@ cleanup_tmpdir (OstreeRepo *self, continue; /* Handle transaction tmpdirs */ - if (_ostree_repo_is_locked_tmpdir (dent->d_name)) + if (_ostree_repo_has_staging_prefix (dent->d_name)) { if (!cleanup_txn_dir (self, dfd_iter.fd, dent->d_name, cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index e52f9f0b..a1c7b7b4 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -247,7 +247,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, GError **error); gboolean -_ostree_repo_is_locked_tmpdir (const char *filename); +_ostree_repo_has_staging_prefix (const char *filename); gboolean _ostree_repo_try_lock_tmpdir (int tmpdir_dfd, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 97ce95b8..5fd92bb9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5959,7 +5959,7 @@ _ostree_repo_maybe_regenerate_summary (OstreeRepo *self, } gboolean -_ostree_repo_is_locked_tmpdir (const char *filename) +_ostree_repo_has_staging_prefix (const char *filename) { return g_str_has_prefix (filename, OSTREE_REPO_TMPDIR_STAGING); } @@ -6019,7 +6019,7 @@ _ostree_repo_allocate_tmpdir (int tmpdir_dfd, GCancellable *cancellable, GError **error) { - g_return_val_if_fail (_ostree_repo_is_locked_tmpdir (tmpdir_prefix), FALSE); + g_return_val_if_fail (_ostree_repo_has_staging_prefix (tmpdir_prefix), FALSE); /* Look for existing tmpdir (with same prefix) to reuse */ g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; From 8ece36c28a3449de9dce9d43252aeb0ffc166687 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 17 Apr 2020 14:29:13 -0400 Subject: [PATCH 113/177] lib/commit: Check that dirent is a directory before cleaning I've only noticed this by inspection. But I think it's possible for `cleanup_txn_dir` to get called with the `staging-...-lock` file since it matches the prefix. Make the checking here stronger by verifying that it's a directory. If it's not a directory (lockfile), then follow the default pruning expiry logic so that we still cleanup stray lockfiles eventually. --- src/libostree/ostree-repo-commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 8c4bd3fe..7dd68e96 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2011,7 +2011,7 @@ cleanup_tmpdir (OstreeRepo *self, continue; /* Handle transaction tmpdirs */ - if (_ostree_repo_has_staging_prefix (dent->d_name)) + if (_ostree_repo_has_staging_prefix (dent->d_name) && S_ISDIR (stbuf.st_mode)) { if (!cleanup_txn_dir (self, dfd_iter.fd, dent->d_name, cancellable, error)) return FALSE; From 12984ddef2d0b9eec923b9a4ea06c9dad870b452 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Apr 2020 01:20:28 +0000 Subject: [PATCH 114/177] lib: Move pull struct definition into repo-pull-private.h Prep for further splitting up `ostree-repo-pull.c`. --- src/libostree/ostree-repo-pull-private.h | 113 +++++++++++++++++++++- src/libostree/ostree-repo-pull.c | 114 +---------------------- 2 files changed, 113 insertions(+), 114 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index 918c4617..0ed0fdff 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -21,9 +21,120 @@ #pragma once -#include "ostree-core.h" +#include "ostree-repo-private.h" +#include "ostree-fetcher-util.h" +#include "ostree-remote-private.h" G_BEGIN_DECLS +typedef enum { + OSTREE_FETCHER_SECURITY_STATE_CA_PINNED, + OSTREE_FETCHER_SECURITY_STATE_TLS, + OSTREE_FETCHER_SECURITY_STATE_INSECURE, +} OstreeFetcherSecurityState; + +typedef struct { + OstreeRepo *repo; + int tmpdir_dfd; + OstreeRepoPullFlags flags; + char *remote_name; + char *remote_refspec_name; + OstreeRepoMode remote_mode; + OstreeFetcher *fetcher; + OstreeFetcherSecurityState fetcher_security_state; + + GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ + GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ + OstreeRepo *remote_repo_local; + GPtrArray *localcache_repos; /* Array */ + + GMainContext *main_context; + GCancellable *cancellable; + OstreeAsyncProgress *progress; + + GVariant *extra_headers; + char *append_user_agent; + + gboolean dry_run; + gboolean dry_run_emitted_progress; + gboolean legacy_transaction_resuming; + guint n_network_retries; + enum { + OSTREE_PULL_PHASE_FETCHING_REFS, + OSTREE_PULL_PHASE_FETCHING_OBJECTS + } phase; + gint n_scanned_metadata; + + gboolean gpg_verify; + gboolean gpg_verify_summary; + gboolean sign_verify; + gboolean sign_verify_summary; + gboolean require_static_deltas; + gboolean disable_static_deltas; + gboolean has_tombstone_commits; + + GBytes *summary_data; + GBytes *summary_data_sig; + GVariant *summary; + GHashTable *summary_deltas_checksums; + GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ + GHashTable *verified_commits; /* Set of commits that have been verified */ + GHashTable *ref_keyring_map; /* Maps OstreeCollectionRef to keyring remote name */ + GPtrArray *static_delta_superblocks; + GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */ + GHashTable *commit_to_depth; /* Maps commit checksum maximum depth */ + GHashTable *scanned_metadata; /* Maps object name to itself */ + GHashTable *fetched_detached_metadata; /* Map */ + GHashTable *requested_metadata; /* Maps object name to itself */ + GHashTable *requested_content; /* Maps checksum to itself */ + GHashTable *requested_fallback_content; /* Maps checksum to itself */ + GHashTable *pending_fetch_metadata; /* Map */ + GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_delta_superblocks; /* Set */ + GHashTable *pending_fetch_deltaparts; /* Set */ + guint n_outstanding_metadata_fetches; + guint n_outstanding_metadata_write_requests; + guint n_outstanding_content_fetches; + guint n_outstanding_content_write_requests; + guint n_outstanding_deltapart_fetches; + guint n_outstanding_deltapart_write_requests; + guint n_total_deltaparts; + guint n_total_delta_fallbacks; + guint64 fetched_deltapart_size; /* How much of the delta we have now */ + guint64 total_deltapart_size; + guint64 total_deltapart_usize; + gint n_requested_metadata; + gint n_requested_content; + guint n_fetched_deltaparts; + guint n_fetched_deltapart_fallbacks; + guint n_fetched_metadata; + guint n_fetched_content; + /* Objects imported via hardlink/reflink/copying or --localcache-repo*/ + guint n_imported_metadata; + guint n_imported_content; + + gboolean timestamp_check; /* Verify commit timestamps */ + int maxdepth; + guint64 max_metadata_size; + guint64 start_time; + + gboolean is_mirror; + gboolean trusted_http_direct; + gboolean is_commit_only; + OstreeRepoImportFlags importflags; + + GPtrArray *dirs; + + gboolean have_previous_bytes; + guint64 previous_bytes_sec; + guint64 previous_total_downloaded; + + GError *cached_async_error; + GError **async_error; + gboolean caught_error; + + GQueue scan_object_queue; + GSource *idle_src; +} OtPullData; G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f376da82..6b8c9e66 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -29,15 +29,13 @@ #include "libglnx.h" #include "ostree.h" #include "otutil.h" -#include "ostree-repo-private.h" +#include "ostree-repo-pull-private.h" #ifdef HAVE_LIBCURL_OR_LIBSOUP #include "ostree-core-private.h" #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" -#include "ostree-fetcher-util.h" -#include "ostree-remote-private.h" #include "ot-fs-utils.h" #include "ostree-repo-finder.h" @@ -66,116 +64,6 @@ * `n-network-retries` pull option. */ #define DEFAULT_N_NETWORK_RETRIES 5 -typedef enum { - OSTREE_FETCHER_SECURITY_STATE_CA_PINNED, - OSTREE_FETCHER_SECURITY_STATE_TLS, - OSTREE_FETCHER_SECURITY_STATE_INSECURE, -} OstreeFetcherSecurityState; - -typedef struct { - OstreeRepo *repo; - int tmpdir_dfd; - OstreeRepoPullFlags flags; - char *remote_name; - char *remote_refspec_name; - OstreeRepoMode remote_mode; - OstreeFetcher *fetcher; - OstreeFetcherSecurityState fetcher_security_state; - - GPtrArray *meta_mirrorlist; /* List of base URIs for fetching metadata */ - GPtrArray *content_mirrorlist; /* List of base URIs for fetching content */ - OstreeRepo *remote_repo_local; - GPtrArray *localcache_repos; /* Array */ - - GMainContext *main_context; - GCancellable *cancellable; - OstreeAsyncProgress *progress; - - GVariant *extra_headers; - char *append_user_agent; - - gboolean dry_run; - gboolean dry_run_emitted_progress; - gboolean legacy_transaction_resuming; - guint n_network_retries; - enum { - OSTREE_PULL_PHASE_FETCHING_REFS, - OSTREE_PULL_PHASE_FETCHING_OBJECTS - } phase; - gint n_scanned_metadata; - - gboolean gpg_verify; - gboolean gpg_verify_summary; - gboolean sign_verify; - gboolean sign_verify_summary; - gboolean require_static_deltas; - gboolean disable_static_deltas; - gboolean has_tombstone_commits; - - GBytes *summary_data; - GBytes *summary_data_sig; - GVariant *summary; - GHashTable *summary_deltas_checksums; - GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ - GHashTable *verified_commits; /* Set of commits that have been verified */ - GHashTable *ref_keyring_map; /* Maps OstreeCollectionRef to keyring remote name */ - GPtrArray *static_delta_superblocks; - GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */ - GHashTable *commit_to_depth; /* Maps commit checksum maximum depth */ - GHashTable *scanned_metadata; /* Maps object name to itself */ - GHashTable *fetched_detached_metadata; /* Map */ - GHashTable *requested_metadata; /* Maps object name to itself */ - GHashTable *requested_content; /* Maps checksum to itself */ - GHashTable *requested_fallback_content; /* Maps checksum to itself */ - GHashTable *pending_fetch_metadata; /* Map */ - GHashTable *pending_fetch_content; /* Map */ - GHashTable *pending_fetch_delta_superblocks; /* Set */ - GHashTable *pending_fetch_deltaparts; /* Set */ - guint n_outstanding_metadata_fetches; - guint n_outstanding_metadata_write_requests; - guint n_outstanding_content_fetches; - guint n_outstanding_content_write_requests; - guint n_outstanding_deltapart_fetches; - guint n_outstanding_deltapart_write_requests; - guint n_total_deltaparts; - guint n_total_delta_fallbacks; - guint64 fetched_deltapart_size; /* How much of the delta we have now */ - guint64 total_deltapart_size; - guint64 total_deltapart_usize; - gint n_requested_metadata; - gint n_requested_content; - guint n_fetched_deltaparts; - guint n_fetched_deltapart_fallbacks; - guint n_fetched_metadata; - guint n_fetched_content; - /* Objects imported via hardlink/reflink/copying or --localcache-repo*/ - guint n_imported_metadata; - guint n_imported_content; - - gboolean timestamp_check; /* Verify commit timestamps */ - int maxdepth; - guint64 max_metadata_size; - guint64 start_time; - - gboolean is_mirror; - gboolean trusted_http_direct; - gboolean is_commit_only; - OstreeRepoImportFlags importflags; - - GPtrArray *dirs; - - gboolean have_previous_bytes; - guint64 previous_bytes_sec; - guint64 previous_total_downloaded; - - GError *cached_async_error; - GError **async_error; - gboolean caught_error; - - GQueue scan_object_queue; - GSource *idle_src; -} OtPullData; - typedef struct { OtPullData *pull_data; GVariant *object; From 1b8fed247cedac3ff8874d2bb017a2c8a6754f31 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Apr 2020 01:13:51 +0000 Subject: [PATCH 115/177] lib: Move gpg/signapi bits into ostree-repo-pull-verify.c `ostree-repo-pull.c` is huge; separate some of the GPG/signing verification functions into their own file so they're more easily seen. --- Makefile-libostree.am | 1 + src/libostree/ostree-repo-pull-private.h | 28 +++ src/libostree/ostree-repo-pull-verify.c | 302 +++++++++++++++++++++++ src/libostree/ostree-repo-pull.c | 282 +-------------------- 4 files changed, 338 insertions(+), 275 deletions(-) create mode 100644 src/libostree/ostree-repo-pull-verify.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index c0a7ac9f..96b9249b 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -95,6 +95,7 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-repo-commit.c \ src/libostree/ostree-repo-pull.c \ src/libostree/ostree-repo-pull-private.h \ + src/libostree/ostree-repo-pull-verify.c \ src/libostree/ostree-repo-libarchive.c \ src/libostree/ostree-repo-prune.c \ src/libostree/ostree-repo-refs.c \ diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index 0ed0fdff..b9eb2342 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -137,4 +137,32 @@ typedef struct { GSource *idle_src; } OtPullData; +gboolean +_sign_verify_for_remote (OstreeRepo *repo, + const gchar *remote_name, + GBytes *signed_data, + GVariant *metadata, + GError **error); + +gboolean +_signapi_load_public_keys (OstreeSign *sign, + OstreeRepo *repo, + const gchar *remote_name, + GError **error); + +gboolean +_verify_unwritten_commit (OtPullData *pull_data, + const char *checksum, + GVariant *commit, + GVariant *detached_metadata, + const OstreeCollectionRef *ref, + GCancellable *cancellable, + GError **error); + +gboolean +_process_gpg_verify_result (OtPullData *pull_data, + const char *checksum, + OstreeGpgVerifyResult *result, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c new file mode 100644 index 00000000..c1eab6c3 --- /dev/null +++ b/src/libostree/ostree-repo-pull-verify.c @@ -0,0 +1,302 @@ +/* + * Copyright (C) 2020 Red Hat, Inc. + * Copyright © 2017 Endless Mobile, Inc. + * + * SPDX-License-Identifier: LGPL-2.0+ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "libglnx.h" +#include "ostree.h" +#include "otutil.h" +#include "ostree-repo-pull-private.h" +#include "ostree-repo-private.h" + +#include "ostree-core-private.h" +#include "ostree-repo-static-delta-private.h" +#include "ostree-metalink.h" +#include "ostree-fetcher-util.h" +#include "ostree-remote-private.h" +#include "ot-fs-utils.h" + +#include +#include +#ifdef HAVE_LIBSYSTEMD +#include +#endif + +#include "ostree-sign.h" + +static gboolean +get_signapi_remote_option (OstreeRepo *repo, + OstreeSign *sign, + const char *remote_name, + const char *keysuffix, + char **out_value, + GError **error) +{ + g_autofree char *key = g_strdup_printf ("verification-%s-%s", ostree_sign_get_name (sign), keysuffix); + return ostree_repo_get_remote_option (repo, remote_name, key, NULL, out_value, error); +} + +/* _signapi_load_public_keys: + * + * Load public keys according remote's configuration: + * inlined key passed via config option `verification--key` or + * file name with public keys via `verification--file` option. + * + * If both options are set then load all all public keys + * both from file and inlined in config. + * + * Returns: %FALSE if any source is configured but nothing has been loaded. + * Returns: %TRUE if no configuration or any key loaded. + * */ +gboolean +_signapi_load_public_keys (OstreeSign *sign, + OstreeRepo *repo, + const gchar *remote_name, + GError **error) +{ + g_autofree gchar *pk_ascii = NULL; + g_autofree gchar *pk_file = NULL; + gboolean loaded_from_file = TRUE; + gboolean loaded_inlined = TRUE; + + if (!get_signapi_remote_option (repo, sign, remote_name, "file", &pk_file, error)) + return FALSE; + if (!get_signapi_remote_option (repo, sign, remote_name, "key", &pk_ascii, error)) + return FALSE; + + /* return TRUE if there is no configuration for remote */ + if ((pk_file == NULL) &&(pk_ascii == NULL)) + { + /* It is expected what remote may have verification file as + * a part of configuration. Hence there is not a lot of sense + * for automatic resolve of per-remote keystore file as it + * used in find_keyring () for GPG. + * If it is needed to add the similar mechanism, it is preferable + * to pass the path to ostree_sign_load_pk () via GVariant options + * and call it here for loading with method and file structure + * specific for signature type. + */ + return TRUE; + } + + if (pk_file != NULL) + { + g_autoptr (GError) local_error = NULL; + g_autoptr (GVariantBuilder) builder = NULL; + g_autoptr (GVariant) options = NULL; + + builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); + g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); + options = g_variant_builder_end (builder); + + if (ostree_sign_load_pk (sign, options, &local_error)) + loaded_from_file = TRUE; + else + { + return glnx_throw (error, "Failed loading '%s' keys from '%s", + ostree_sign_get_name (sign), pk_file); + } + } + + if (pk_ascii != NULL) + { + g_autoptr (GError) local_error = NULL; + g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); + + /* Add inlined public key */ + if (loaded_from_file) + loaded_inlined = ostree_sign_add_pk (sign, pk, &local_error); + else + loaded_inlined = ostree_sign_set_pk (sign, pk, &local_error); + + if (!loaded_inlined) + { + return glnx_throw (error, "Failed loading '%s' keys from inline `verification-key`", + ostree_sign_get_name (sign)); + } + } + + /* Return true if able to load from any source */ + if (!(loaded_from_file || loaded_inlined)) + return glnx_throw (error, "No keys found"); + + return TRUE; +} + +gboolean +_sign_verify_for_remote (OstreeRepo *repo, + const gchar *remote_name, + GBytes *signed_data, + GVariant *metadata, + GError **error) +{ + /* list all signature types in detached metadata and check if signed by any? */ + g_auto (GStrv) names = ostree_sign_list_names(); + guint n_invalid_signatures = 0; + guint n_unknown_signatures = 0; + g_autoptr (GError) last_sig_error = NULL; + gboolean found_sig = FALSE; + + for (char **iter=names; iter && *iter; iter++) + { + g_autoptr (OstreeSign) sign = NULL; + g_autoptr (GVariant) signatures = NULL; + const gchar *signature_key = NULL; + GVariantType *signature_format = NULL; + + if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) + { + n_unknown_signatures++; + continue; + } + + signature_key = ostree_sign_metadata_key (sign); + signature_format = (GVariantType *) ostree_sign_metadata_format (sign); + + signatures = g_variant_lookup_value (metadata, + signature_key, + signature_format); + + /* If not found signatures for requested signature subsystem */ + if (!signatures) + continue; + + /* Try to load public key(s) according remote's configuration */ + if (!_signapi_load_public_keys (sign, repo, remote_name, error)) + return FALSE; + + found_sig = TRUE; + + /* Return true if any signature fit to pre-loaded public keys. + * If no keys configured -- then system configuration will be used */ + if (!ostree_sign_data_verify (sign, + signed_data, + signatures, + last_sig_error ? NULL : &last_sig_error)) + { + n_invalid_signatures++; + continue; + } + /* Accept the first valid signature */ + return TRUE; + } + + if (!found_sig) + { + if (n_unknown_signatures > 0) + return glnx_throw (error, "No signatures found (%d unknown type)", n_unknown_signatures); + return glnx_throw (error, "No signatures found"); + } + + g_assert (last_sig_error); + g_propagate_error (error, g_steal_pointer (&last_sig_error)); + if (n_invalid_signatures > 1) + glnx_prefix_error (error, "(%d other invalid signatures)", n_invalid_signatures-1); + return FALSE; +} + + +#ifndef OSTREE_DISABLE_GPGME +gboolean +_process_gpg_verify_result (OtPullData *pull_data, + const char *checksum, + OstreeGpgVerifyResult *result, + GError **error) +{ + const char *error_prefix = glnx_strjoina ("Commit ", checksum); + GLNX_AUTO_PREFIX_ERROR(error_prefix, error); + if (result == NULL) + return FALSE; + + /* Allow callers to output the results immediately. */ + g_signal_emit_by_name (pull_data->repo, + "gpg-verify-result", + checksum, result); + + if (!ostree_gpg_verify_result_require_valid_signature (result, error)) + return FALSE; + + + /* We now check both *before* writing the commit, and after. Because the + * behavior used to be only verifiying after writing, we need to handle + * the case of "written but not verified". But we also don't want to check + * twice, as that'd result in duplicate signals. + */ + g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); + + return TRUE; +} +#endif /* OSTREE_DISABLE_GPGME */ + +gboolean +_verify_unwritten_commit (OtPullData *pull_data, + const char *checksum, + GVariant *commit, + GVariant *detached_metadata, + const OstreeCollectionRef *ref, + GCancellable *cancellable, + GError **error) +{ + + if (pull_data->gpg_verify || pull_data->sign_verify) + /* Shouldn't happen, but see comment in process_gpg_verify_result() */ + if (g_hash_table_contains (pull_data->verified_commits, checksum)) + return TRUE; + + g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); + +#ifndef OSTREE_DISABLE_GPGME + if (pull_data->gpg_verify) + { + const char *keyring_remote = NULL; + + if (ref != NULL) + keyring_remote = g_hash_table_lookup (pull_data->ref_keyring_map, ref); + if (keyring_remote == NULL) + keyring_remote = pull_data->remote_name; + + g_autoptr(OstreeGpgVerifyResult) result = + _ostree_repo_gpg_verify_with_metadata (pull_data->repo, signed_data, + detached_metadata, + keyring_remote, + NULL, NULL, cancellable, error); + if (!_process_gpg_verify_result (pull_data, checksum, result, error)) + return FALSE; + } +#endif /* OSTREE_DISABLE_GPGME */ + + if (pull_data->sign_verify) + { + /* Nothing to check if detached metadata is absent */ + if (detached_metadata == NULL) + return glnx_throw (error, "Can't verify commit without detached metadata"); + + if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) + return glnx_prefix_error (error, "Can't verify commit"); + + /* Mark the commit as verified to avoid double verification + * see process_verify_result () for rationale */ + g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); + } + + return TRUE; +} diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 6b8c9e66..817307e9 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -36,7 +36,6 @@ #include "ostree-core-private.h" #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" -#include "ot-fs-utils.h" #include "ostree-repo-finder.h" #include "ostree-repo-finder-config.h" @@ -51,8 +50,6 @@ #include #endif -#include "ostree-sign.h" - #define OSTREE_MESSAGE_FETCH_COMPLETE_ID SD_ID128_MAKE(75,ba,3d,eb,0a,f0,41,a9,a4,62,72,ff,85,d9,e7,3e) #define OSTREE_REPO_PULL_CONTENT_PRIORITY (OSTREE_FETCHER_DEFAULT_PRIORITY) @@ -151,14 +148,6 @@ static gboolean scan_one_metadata_object (OtPullData *pull_data, GCancellable *cancellable, GError **error); static void scan_object_queue_data_free (ScanObjectQueueData *scan_data); -static gboolean -ostree_verify_unwritten_commit (OtPullData *pull_data, - const char *checksum, - GVariant *commit, - GVariant *detached_metadata, - const OstreeCollectionRef *ref, - GCancellable *cancellable, - GError **error); static gboolean update_progress (gpointer user_data) @@ -1200,8 +1189,8 @@ meta_fetch_on_complete (GObject *object, * metadata into this hash. */ GVariant *detached_data = g_hash_table_lookup (pull_data->fetched_detached_metadata, checksum); - if (!ostree_verify_unwritten_commit (pull_data, checksum, metadata, detached_data, - fetch_data->requested_ref, pull_data->cancellable, error)) + if (!_verify_unwritten_commit (pull_data, checksum, metadata, detached_data, + fetch_data->requested_ref, pull_data->cancellable, error)) goto out; if (!ostree_repo_mark_commit_partial (pull_data->repo, checksum, TRUE, error)) @@ -1325,263 +1314,6 @@ static_deltapart_fetch_on_complete (GObject *object, g_clear_pointer (&fetch_data, fetch_static_delta_data_free); } -#ifndef OSTREE_DISABLE_GPGME -static gboolean -process_gpg_verify_result (OtPullData *pull_data, - const char *checksum, - OstreeGpgVerifyResult *result, - GError **error) -{ - const char *error_prefix = glnx_strjoina ("Commit ", checksum); - GLNX_AUTO_PREFIX_ERROR(error_prefix, error); - if (result == NULL) - return FALSE; - - /* Allow callers to output the results immediately. */ - g_signal_emit_by_name (pull_data->repo, - "gpg-verify-result", - checksum, result); - - if (!ostree_gpg_verify_result_require_valid_signature (result, error)) - return FALSE; - - - /* We now check both *before* writing the commit, and after. Because the - * behavior used to be only verifiying after writing, we need to handle - * the case of "written but not verified". But we also don't want to check - * twice, as that'd result in duplicate signals. - */ - g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); - - return TRUE; -} -#endif /* OSTREE_DISABLE_GPGME */ - -static gboolean -get_signapi_remote_option (OstreeRepo *repo, - OstreeSign *sign, - const char *remote_name, - const char *keysuffix, - char **out_value, - GError **error) -{ - g_autofree char *key = g_strdup_printf ("verification-%s-%s", ostree_sign_get_name (sign), keysuffix); - return ostree_repo_get_remote_option (repo, remote_name, key, NULL, out_value, error); -} - -/* _load_public_keys: - * - * Load public keys according remote's configuration: - * inlined key passed via config option `verification--key` or - * file name with public keys via `verification--file` option. - * - * If both options are set then load all all public keys - * both from file and inlined in config. - * - * Returns: %FALSE if any source is configured but nothing has been loaded. - * Returns: %TRUE if no configuration or any key loaded. - * */ -static gboolean -_load_public_keys (OstreeSign *sign, - OstreeRepo *repo, - const gchar *remote_name, - GError **error) -{ - g_autofree gchar *pk_ascii = NULL; - g_autofree gchar *pk_file = NULL; - gboolean loaded_from_file = TRUE; - gboolean loaded_inlined = TRUE; - - if (!get_signapi_remote_option (repo, sign, remote_name, "file", &pk_file, error)) - return FALSE; - if (!get_signapi_remote_option (repo, sign, remote_name, "key", &pk_ascii, error)) - return FALSE; - - /* return TRUE if there is no configuration for remote */ - if ((pk_file == NULL) &&(pk_ascii == NULL)) - { - /* It is expected what remote may have verification file as - * a part of configuration. Hence there is not a lot of sense - * for automatic resolve of per-remote keystore file as it - * used in find_keyring () for GPG. - * If it is needed to add the similar mechanism, it is preferable - * to pass the path to ostree_sign_load_pk () via GVariant options - * and call it here for loading with method and file structure - * specific for signature type. - */ - return TRUE; - } - - if (pk_file != NULL) - { - g_autoptr (GError) local_error = NULL; - g_autoptr (GVariantBuilder) builder = NULL; - g_autoptr (GVariant) options = NULL; - - builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); - g_variant_builder_add (builder, "{sv}", "filename", g_variant_new_string (pk_file)); - options = g_variant_builder_end (builder); - - if (ostree_sign_load_pk (sign, options, &local_error)) - loaded_from_file = TRUE; - else - { - return glnx_throw (error, "Failed loading '%s' keys from '%s", - ostree_sign_get_name (sign), pk_file); - } - } - - if (pk_ascii != NULL) - { - g_autoptr (GError) local_error = NULL; - g_autoptr (GVariant) pk = g_variant_new_string(pk_ascii); - - /* Add inlined public key */ - if (loaded_from_file) - loaded_inlined = ostree_sign_add_pk (sign, pk, &local_error); - else - loaded_inlined = ostree_sign_set_pk (sign, pk, &local_error); - - if (!loaded_inlined) - { - return glnx_throw (error, "Failed loading '%s' keys from inline `verification-key`", - ostree_sign_get_name (sign)); - } - } - - /* Return true if able to load from any source */ - if (!(loaded_from_file || loaded_inlined)) - return glnx_throw (error, "No keys found"); - - return TRUE; -} - -static gboolean -_sign_verify_for_remote (OstreeRepo *repo, - const gchar *remote_name, - GBytes *signed_data, - GVariant *metadata, - GError **error) -{ - /* list all signature types in detached metadata and check if signed by any? */ - g_auto (GStrv) names = ostree_sign_list_names(); - guint n_invalid_signatures = 0; - guint n_unknown_signatures = 0; - g_autoptr (GError) last_sig_error = NULL; - gboolean found_sig = FALSE; - - for (char **iter=names; iter && *iter; iter++) - { - g_autoptr (OstreeSign) sign = NULL; - g_autoptr (GVariant) signatures = NULL; - const gchar *signature_key = NULL; - GVariantType *signature_format = NULL; - - if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) - { - n_unknown_signatures++; - continue; - } - - signature_key = ostree_sign_metadata_key (sign); - signature_format = (GVariantType *) ostree_sign_metadata_format (sign); - - signatures = g_variant_lookup_value (metadata, - signature_key, - signature_format); - - /* If not found signatures for requested signature subsystem */ - if (!signatures) - continue; - - /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (sign, repo, remote_name, error)) - return FALSE; - - found_sig = TRUE; - - /* Return true if any signature fit to pre-loaded public keys. - * If no keys configured -- then system configuration will be used */ - if (!ostree_sign_data_verify (sign, - signed_data, - signatures, - last_sig_error ? NULL : &last_sig_error)) - { - n_invalid_signatures++; - continue; - } - /* Accept the first valid signature */ - return TRUE; - } - - if (!found_sig) - { - if (n_unknown_signatures > 0) - return glnx_throw (error, "No signatures found (%d unknown type)", n_unknown_signatures); - return glnx_throw (error, "No signatures found"); - } - - g_assert (last_sig_error); - g_propagate_error (error, g_steal_pointer (&last_sig_error)); - if (n_invalid_signatures > 1) - glnx_prefix_error (error, "(%d other invalid signatures)", n_invalid_signatures-1); - return FALSE; -} - -static gboolean -ostree_verify_unwritten_commit (OtPullData *pull_data, - const char *checksum, - GVariant *commit, - GVariant *detached_metadata, - const OstreeCollectionRef *ref, - GCancellable *cancellable, - GError **error) -{ - - if (pull_data->gpg_verify || pull_data->sign_verify) - /* Shouldn't happen, but see comment in process_gpg_verify_result() */ - if (g_hash_table_contains (pull_data->verified_commits, checksum)) - return TRUE; - - g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); - -#ifndef OSTREE_DISABLE_GPGME - if (pull_data->gpg_verify) - { - const char *keyring_remote = NULL; - - if (ref != NULL) - keyring_remote = g_hash_table_lookup (pull_data->ref_keyring_map, ref); - if (keyring_remote == NULL) - keyring_remote = pull_data->remote_name; - - g_autoptr(OstreeGpgVerifyResult) result = - _ostree_repo_gpg_verify_with_metadata (pull_data->repo, signed_data, - detached_metadata, - keyring_remote, - NULL, NULL, cancellable, error); - if (!process_gpg_verify_result (pull_data, checksum, result, error)) - return FALSE; - } -#endif /* OSTREE_DISABLE_GPGME */ - - if (pull_data->sign_verify) - { - /* Nothing to check if detached metadata is absent */ - if (detached_metadata == NULL) - return glnx_throw (error, "Can't verify commit without detached metadata"); - - if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) - return glnx_prefix_error (error, "Can't verify commit"); - - /* Mark the commit as verified to avoid double verification - * see process_verify_result () for rationale */ - g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); - } - - return TRUE; -} - static gboolean commitstate_is_partial (OtPullData *pull_data, OstreeRepoCommitState commitstate) @@ -1800,7 +1532,7 @@ scan_commit_object (OtPullData *pull_data, keyring_remote, cancellable, error); - if (!process_gpg_verify_result (pull_data, checksum, result, error)) + if (!_process_gpg_verify_result (pull_data, checksum, result, error)) return FALSE; } #endif /* OSTREE_DISABLE_GPGME */ @@ -1822,7 +1554,7 @@ scan_commit_object (OtPullData *pull_data, continue; /* Try to load public key(s) according remote's configuration */ - if (!_load_public_keys (sign, pull_data->repo, pull_data->remote_name, error)) + if (!_signapi_load_public_keys (sign, pull_data->repo, pull_data->remote_name, error)) return FALSE; found_any_signature = TRUE; @@ -2434,8 +2166,8 @@ process_one_static_delta (OtPullData *pull_data, g_autofree char *detached_path = _ostree_get_relative_static_delta_path (from_revision, to_revision, "commitmeta"); g_autoptr(GVariant) detached_data = g_variant_lookup_value (metadata, detached_path, G_VARIANT_TYPE("a{sv}")); - if (!ostree_verify_unwritten_commit (pull_data, to_revision, to_commit, detached_data, - ref, cancellable, error)) + if (!_verify_unwritten_commit (pull_data, to_revision, to_commit, detached_data, + ref, cancellable, error)) return FALSE; if (detached_data && !ostree_repo_write_commit_detached_metadata (pull_data->repo, @@ -3664,7 +3396,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->gpg_verify || pull_data->gpg_verify_summary) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, - "'%s': GPG feature is disabled in a build time", + "'%s': GPG feature is disabled at build time", __FUNCTION__); goto out; } From 368574b6577c9039887e207be1c0501d4c4ef8d0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 21 Apr 2020 19:25:40 +0000 Subject: [PATCH 116/177] deploy: Add --no-merge All of the underlying libostree APIs have supported passing `NULL` for a merge deployment for...a long time. But we never plumbed it up into the CLI. Add a `--no-merge` option to aid people who want to do a "factory reset": https://github.com/ostreedev/ostree/issues/1793 --- .cci.jenkinsfile | 1 + Makefile-tests.am | 1 + src/ostree/ot-admin-builtin-deploy.c | 4 ++- tests/test-admin-deploy-nomerge.sh | 42 ++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 tests/test-admin-deploy-nomerge.sh diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 484c3b14..d1f2a231 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -74,6 +74,7 @@ parallel fcos: { mkdir -p overrides/rootfs mv insttree/* overrides/rootfs/ rmdir insttree + coreos-assembler fetch coreos-assembler build """) } diff --git a/Makefile-tests.am b/Makefile-tests.am index 3270bd9c..a172f015 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -102,6 +102,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-admin-deploy-etcmerge-cornercases.sh \ tests/test-admin-deploy-uboot.sh \ tests/test-admin-deploy-grub2.sh \ + tests/test-admin-deploy-nomerge.sh \ tests/test-admin-deploy-none.sh \ tests/test-admin-deploy-bootid-gc.sh \ tests/test-admin-instutil-set-kargs.sh \ diff --git a/src/ostree/ot-admin-builtin-deploy.c b/src/ostree/ot-admin-builtin-deploy.c index c1c3353d..bcece3f6 100644 --- a/src/ostree/ot-admin-builtin-deploy.c +++ b/src/ostree/ot-admin-builtin-deploy.c @@ -37,6 +37,7 @@ static gboolean opt_retain_pending; static gboolean opt_retain_rollback; static gboolean opt_not_as_default; static gboolean opt_no_prune; +static gboolean opt_no_merge; static char **opt_kernel_argv; static char **opt_kernel_argv_append; static gboolean opt_kernel_proc_cmdline; @@ -48,6 +49,7 @@ static GOptionEntry options[] = { { "os", 0, 0, G_OPTION_ARG_STRING, &opt_osname, "Use a different operating system root than the current one", "OSNAME" }, { "origin-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_origin_path, "Specify origin file", "FILENAME" }, { "no-prune", 0, 0, G_OPTION_ARG_NONE, &opt_no_prune, "Don't prune the repo when done", NULL}, + { "no-merge", 0, 0, G_OPTION_ARG_NONE, &opt_no_merge, "Do not apply configuration (/etc and kernel arguments) from booted deployment", NULL}, { "retain", 0, 0, G_OPTION_ARG_NONE, &opt_retain, "Do not delete previous deployments", NULL }, { "stage", 0, 0, G_OPTION_ARG_NONE, &opt_stage, "Complete deployment at OS shutdown", NULL }, { "retain-pending", 0, 0, G_OPTION_ARG_NONE, &opt_retain_pending, "Do not delete pending deployments", NULL }, @@ -113,7 +115,7 @@ ot_admin_builtin_deploy (int argc, char **argv, OstreeCommandInvocation *invocat return FALSE; g_autoptr(OstreeDeployment) merge_deployment = - ostree_sysroot_get_merge_deployment (sysroot, opt_osname); + opt_no_merge ? NULL : ostree_sysroot_get_merge_deployment (sysroot, opt_osname); /* Here we perform cleanup of any leftover data from previous * partial failures. This avoids having to call diff --git a/tests/test-admin-deploy-nomerge.sh b/tests/test-admin-deploy-nomerge.sh new file mode 100755 index 00000000..06271421 --- /dev/null +++ b/tests/test-admin-deploy-nomerge.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Copyright (C) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +# Exports OSTREE_SYSROOT so --sysroot not needed. +setup_os_repository "archive" "syslinux" +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testos-repo testos/buildmaster/x86_64-runtime + +echo "1..1" +${CMD_PREFIX} ostree admin deploy --os=testos --karg=root=LABEL=foo --karg=testkarg=1 testos:testos/buildmaster/x86_64-runtime +origdeployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir) +testconfig=etc/modified-config-file-that-will-be-removed +touch "${origdeployment}"/"${testconfig}" +assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf "^options.*root=LABEL=foo.*testkarg" +${CMD_PREFIX} ostree admin deploy --os=testos --no-merge --karg=root=LABEL=bar testos:testos/buildmaster/x86_64-runtime +deployment=$(${CMD_PREFIX} ostree admin --sysroot=sysroot --print-current-dir) +assert_not_streq "${origdeployment}" "${deployment}" +assert_not_has_file "${deployment}/${testconfig}" +assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf "^options root=LABEL=bar" +assert_not_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf "^options .*testkarg" +echo "ok no merge deployment" From 47a3096ab8130e7306069faefeba9c02e5171d3c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 22 Apr 2020 14:26:23 +0000 Subject: [PATCH 117/177] finalize-staged: Add ProtectHome=yes and ReadOnlyPaths=/etc Same motivation as https://github.com/coreos/rpm-ostree/pull/2060 I tried `InaccessiblePaths=/var` first and was very sad to find out we have one tiny exception that breaks it. Otherwise it'd be so elegant. Maybe in the future we split out that one thing to a separate `ostree-finalized-stage-var.service` that's just `ExecStart=/bin/rm -vf /var/.updated` and is otherwise `ProtectSystem=strict` etc. --- src/boot/ostree-finalize-staged.service | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/boot/ostree-finalize-staged.service b/src/boot/ostree-finalize-staged.service index 9c4706e8..8152e596 100644 --- a/src/boot/ostree-finalize-staged.service +++ b/src/boot/ostree-finalize-staged.service @@ -39,3 +39,11 @@ ExecStop=/usr/bin/ostree admin finalize-staged # here is that people don't get an upgrade. We need to handle # cases with slow rotational media, etc. TimeoutStopSec=5m +# OSTree should never touch /var at all...except, we need to remove +# the /var/.updated flag, so we can't just `InaccessiblePaths=/var` right now. +# For now, let's at least use ProtectHome just so we have some sandboxing +# of that. +ProtectHome=yes +# And we shouldn't affect the current deployment's /etc. +ReadOnlyPaths=/etc +# We write to /sysroot and /boot of course. From b7662aaf3345f17e234c2ba868b526a3c1baad4d Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Fri, 24 Apr 2020 13:02:05 +0200 Subject: [PATCH 118/177] docs: clarify archive repo type Today `archive-z2` is still used as the default string to indicate a `archive` type repository. Make clear that this is the way it is intended. Otherwise users might think they use an no longer supported OSTree repository since the mode string is still `archive-z2`. Signed-off-by: Stefan Agner --- docs/manual/formats.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/manual/formats.md b/docs/manual/formats.md index 1a669bb1..884b1b5e 100644 --- a/docs/manual/formats.md +++ b/docs/manual/formats.md @@ -22,8 +22,9 @@ where file/content objects are checksummed and managed individually. The `archive` format simply gzip-compresses each content object. Metadata objects are stored uncompressed. This means that it's easy -to serve via static HTTP. Note: this format used to be called `archive-z2` -for historical reasons. +to serve via static HTTP. Note: the repo config file still uses the +historical term `archive-z2` as mode. But this essentially indicates +the modern `archive` format. When you commit new content, you will see new `.filez` files appearing in `objects/`. From b43c0be3471feb19b1718da7ef3aa44a91b36393 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Fri, 24 Apr 2020 13:05:15 +0200 Subject: [PATCH 119/177] docs: extend object type documentation Extend the object type documentation with file endings used for the individual type. Also clarify in which situation content type objects are used and why they do not match the SHA256 hash today. Signed-off-by: Stefan Agner --- docs/manual/repo.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index 08fd5fe2..b8c3ea54 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -31,13 +31,16 @@ regenerate it from source code. A dirtree contains a sorted array of (filename, checksum) pairs for content objects, and a second sorted array of (filename, dirtree checksum, dirmeta checksum), which are -subdirectories. +subdirectories. These type of objects are stored as files +ending with `.dirtree` in the objects directory. ### Dirmeta objects In git, tree objects contain the metadata such as permissions for their children. But OSTree splits this into a separate object to avoid duplicating extended attribute listings. +These type of objects are stored as files ending with `.dirmeta` +in the objects directory. ### Content objects @@ -45,7 +48,13 @@ Unlike the first three object types which are metadata, designed to be `mmap()`ed, the content object has a separate internal header and payload sections. The header contains uid, gid, mode, and symbolic link target (for symlinks), as well as extended attributes. After the -header, for regular files, the content follows. +header, for regular files, the content follows. These parts toghether +form the SHA256 hash for content objects. The content type objects in +this format exist only in `archive` OSTree repositories. Today the +content part is gzip'ed and the objects are stored as files ending +with `.filez` in the objects directory. Because the SHA256 hash is +formed over the uncompressed content, these files do not match the +hash they are named as. The OSTree data format intentionally does not contain timestamps. The reasoning is that data files may be downloaded at different times, and by different build From ce5dfadbd7fa7ba00c4964bb86aa1e3579d0a911 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Fri, 24 Apr 2020 13:12:47 +0200 Subject: [PATCH 120/177] docs: extend repository types Clarify where metadata are stored exactly in the `bare-user` case. Make the first sentence of `bare-user` and `bare-user-only` paragraph symetric to make it easier to jump to the right paragraph for readers in a hury. Stree out that `bare-user-only` may loose metadata. Signed-off-by: Stefan Agner --- docs/manual/repo.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index b8c3ea54..07cc1a93 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -82,20 +82,21 @@ designed to be the source of a "hardlink farm", where each operating system checkout is merely links into it. If you want to store files owned by e.g. root in this mode, you must run OSTree as root. -The `bare-user` is a later addition that is like `bare` in that files -are unpacked, but it can (and should generally) be created as +The `bare-user` mode is a later addition that is like `bare` in that +files are unpacked, but it can (and should generally) be created as non-root. In this mode, extended metadata such as owner uid, gid, and -extended attributes are stored but not actually applied. +extended attributes are stored in extended attributes under the name +`user.ostreemeta` but not actually applied. The `bare-user` mode is useful for build systems that run as non-root but want to generate root-owned content, as well as non-root container systems. -There is a variant to the `bare-user` mode called `bare-user-only`. Unlike +The `bare-user-only` mode is a variant to the `bare-user` mode. Unlike `bare-user`, neither ownership nor extended attributes are stored. These repos are meant to to be checked out in user mode (with the `-U` flag), where this -information is not applied anyway. The main advantage of `bare-user-only` is -that repos can be stored on filesystems which do not support extended -attributes, such as tmpfs. +information is not applied anyway. Hence this mode may loose metadata. +The main advantage of `bare-user-only` is that repos can be stored on +filesystems which do not support extended attributes, such as tmpfs. In contrast, the `archive` mode is designed for serving via plain HTTP. Like tar files, it can be read/written by non-root users. From ee5d0f350f522604d6f068b8a053fc5c6d896e5d Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Fri, 24 Apr 2020 16:34:50 +0000 Subject: [PATCH 121/177] signapi: expose metadata format and key Explicitly expose functions for querying the metadata format and key name used by OstreeSign object: - ostree_sign_metadata_format - ostree_sign_metadata_key This allows to use the same metadata format and key name by 3-rd party applications using signapi. Signed-off-by: Denis Pynkin --- apidoc/ostree-sections.txt | 2 ++ src/libostree/libostree-devel.sym | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 4dcdc482..5586262a 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -719,6 +719,8 @@ ostree_sign_get_name ostree_sign_add_pk ostree_sign_clear_keys ostree_sign_load_pk +ostree_sign_metadata_format +ostree_sign_metadata_key ostree_sign_set_pk ostree_sign_set_sk ostree_sign_summary diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 4348ab8d..9c6157a8 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -32,6 +32,8 @@ global: ostree_sign_get_name; ostree_sign_clear_keys; ostree_sign_load_pk; + ostree_sign_metadata_format; + ostree_sign_metadata_key; ostree_sign_set_pk; ostree_sign_add_pk; ostree_sign_set_sk; From 37e1921b51a970b8b8bfbb0756ad83d8a9836f45 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 29 Apr 2020 01:45:48 +0000 Subject: [PATCH 122/177] tests/staged-deploy: Cleanup initial state I'm using [cosa build-fast](https://github.com/coreos/coreos-assembler/pull/1371) and this test doesn't like starting out with two deployments. Clean things up to one at the start just to simplify things. --- tests/kola/destructive/staged-deploy.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/kola/destructive/staged-deploy.sh b/tests/kola/destructive/staged-deploy.sh index e0c17a27..cc3e7a7e 100755 --- a/tests/kola/destructive/staged-deploy.sh +++ b/tests/kola/destructive/staged-deploy.sh @@ -9,6 +9,10 @@ prepare_tmpdir n=$(nth_boot) case "${n}" in 1) + # Initial cleanup to handle the cosa fast-build case + ## TODO remove workaround for https://github.com/coreos/rpm-ostree/pull/2021 + mkdir -p /var/lib/rpm-ostree/history + rpm-ostree cleanup -pr commit=${host_commit} # Test the deploy --stage functionality; first, we stage a deployment # reboot, and validate that it worked. @@ -71,8 +75,6 @@ case "${n}" in firstdeploycommit=$(rpm-ostree status |grep 'Commit:' |head -1|sed -e 's,^ *Commit: *,,') assert_streq "${firstdeploycommit}" "${newcommit}" # Cleanup - ## TODO remove workaround for https://github.com/coreos/rpm-ostree/pull/2021 - mkdir -p /var/lib/rpm-ostree/history rpm-ostree cleanup -rp echo "ok upgrade with staging" From dc69f56de6dab66f7bb4fe66aa203e84efa9676c Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 30 Apr 2020 14:28:20 -0600 Subject: [PATCH 123/177] lib: Coerce flags enums to GIR bitfields The GI scanner decides if an `enum` is really a `bitfield` if it finds any values that have left shifts. With an `enumeration`, the introspecting language may error or convert to a different type if the user tries to combine values. Change all Flags `enum`s to use left-shifted values so that they're represented as `bitfield`s in the GIR. The primary bug here is that you can't combine `REFS_ONLY` and `NO_PRUNE` when calling `OSTree.Repo.prune()` from an introspected language. This is an IABI break since the typelib will change from `enumeration` to `bitfield`. `OstreeRepoImportFlags` is internal but the change is included here to prepare for a subsequent name that would require bit shifting to operate correctly as a flag. --- src/libostree/ostree-gpg-verify-result.h | 2 +- src/libostree/ostree-repo-private.h | 6 +++--- src/libostree/ostree-repo.h | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-gpg-verify-result.h b/src/libostree/ostree-gpg-verify-result.h index 6f51ce8a..8f243fcd 100644 --- a/src/libostree/ostree-gpg-verify-result.h +++ b/src/libostree/ostree-gpg-verify-result.h @@ -134,7 +134,7 @@ GVariant * ostree_gpg_verify_result_get_all (OstreeGpgVerifyResult *result, * for future variations. **/ typedef enum { - OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT = 0 + OSTREE_GPG_SIGNATURE_FORMAT_DEFAULT = (0 << 0), } OstreeGpgSignatureFormatFlags; _OSTREE_PUBLIC diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a1c7b7b4..a744c069 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -374,9 +374,9 @@ _ostree_repo_verify_commit_internal (OstreeRepo *self, #endif /* OSTREE_DISABLE_GPGME */ typedef enum { - _OSTREE_REPO_IMPORT_FLAGS_NONE, - _OSTREE_REPO_IMPORT_FLAGS_TRUSTED, - _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY, + _OSTREE_REPO_IMPORT_FLAGS_NONE = 0, + _OSTREE_REPO_IMPORT_FLAGS_TRUSTED = (1 << 0), + _OSTREE_REPO_IMPORT_FLAGS_VERIFY_BAREUSERONLY = (1 << 1), } OstreeRepoImportFlags; gboolean diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 1027ac77..e28af29c 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1174,9 +1174,9 @@ void ostree_repo_commit_traverse_iter_cleanup (void *p); * @OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY: Do not traverse individual commit objects, only follow refs */ typedef enum { - OSTREE_REPO_PRUNE_FLAGS_NONE, - OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE, - OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY + OSTREE_REPO_PRUNE_FLAGS_NONE = 0, + OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE = (1 << 0), + OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY = (1 << 1), } OstreeRepoPruneFlags; _OSTREE_PUBLIC From 811082063c0dfd895a2c01d446af1411951f285b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 3 May 2020 17:04:27 +0000 Subject: [PATCH 124/177] signing: Add #define OSTREE_SIGN_NAME_ED25519 Using `#define` or constants instead of strings helps avoid typos and encourages documentation. --- src/libostree/ostree-sign.c | 2 +- src/libostree/ostree-sign.h | 6 ++++++ src/ostree/ot-builtin-commit.c | 3 +-- src/ostree/ot-builtin-sign.c | 3 +-- src/ostree/ot-builtin-summary.c | 3 +-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 281fabc8..68447da6 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -62,7 +62,7 @@ typedef struct _sign_type sign_types[] = { #if defined(HAVE_LIBSODIUM) - {"ed25519", 0}, + {OSTREE_SIGN_NAME_ED25519, 0}, #endif {"dummy", 0} }; diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 678f182d..da10469b 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -49,6 +49,12 @@ static inline gboolean OSTREE_IS_SIGN (gpointer ptr) { return G_TYPE_CHECK_INSTA static inline OstreeSignInterface *OSTREE_SIGN_GET_IFACE (gpointer ptr) { return G_TYPE_INSTANCE_GET_INTERFACE (ptr, ostree_sign_get_type (), OstreeSignInterface); } G_GNUC_END_IGNORE_DEPRECATIONS +/** + * OSTREE_SIGN_NAME_ED25519: + * The name of the default ed25519 signing type. + */ +#define OSTREE_SIGN_NAME_ED25519 "ed25519" + /* Have to use glib-2.44 for this _OSTREE_PUBLIC G_DECLARE_INTERFACE (OstreeSign, ostree_sign, OSTREE, SIGN, GObject) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 72fa2841..88a5d4f4 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -868,8 +868,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio if (opt_key_ids) { /* Initialize crypto system */ - if (!opt_sign_name) - opt_sign_name = "ed25519"; + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; sign = ostree_sign_get_by_name (opt_sign_name, error); if (sign == NULL) diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index 73561b43..d6cc167a 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -105,8 +105,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, goto out; /* Initialize crypto system */ - if (!opt_sign_name) - opt_sign_name = "ed25519"; + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; sign = ostree_sign_get_by_name (opt_sign_name, error); if (sign == NULL) diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c index de6df835..0938f11e 100644 --- a/src/ostree/ot-builtin-summary.c +++ b/src/ostree/ot-builtin-summary.c @@ -103,8 +103,7 @@ ostree_builtin_summary (int argc, char **argv, OstreeCommandInvocation *invocati /* Initialize crypto system */ if (opt_key_ids) { - if (!opt_sign_name) - opt_sign_name = "ed25519"; + opt_sign_name = opt_sign_name ?: OSTREE_SIGN_NAME_ED25519; sign = ostree_sign_get_by_name (opt_sign_name, error); if (sign == NULL) From 7392259332e00c33ed45b904deabde08f4da3e3c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 May 2020 18:31:53 +0000 Subject: [PATCH 125/177] commit: Add --mode-ro-executables option I think we should encourage removing the writable bits from executables. This has happened to me: https://thomask.sdf.org/blog/2019/11/09/take-care-editing-bash-scripts.html And not having the writable bit may help prevent hardlink corruption with OSTree in some cases. We can't do this by default, but add a convenient CLI flag for it. --- src/ostree/ot-builtin-commit.c | 13 +++++++++++-- tests/basic-test.sh | 20 +++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 88a5d4f4..253a58f7 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -56,6 +56,7 @@ static gboolean opt_no_xattrs; static char *opt_selinux_policy; static gboolean opt_selinux_policy_from_base; static gboolean opt_canonical_permissions; +static gboolean opt_ro_executables; static gboolean opt_consume; static gboolean opt_devino_canonical; static char *opt_base; @@ -111,6 +112,7 @@ static GOptionEntry options[] = { { "owner-uid", 0, 0, G_OPTION_ARG_INT, &opt_owner_uid, "Set file ownership user id", "UID" }, { "owner-gid", 0, 0, G_OPTION_ARG_INT, &opt_owner_gid, "Set file ownership group id", "GID" }, { "canonical-permissions", 0, 0, G_OPTION_ARG_NONE, &opt_canonical_permissions, "Canonicalize permissions in the same way bare-user does for hardlinked files", NULL }, + { "mode-ro-executables", 0, 0, G_OPTION_ARG_NONE, &opt_ro_executables, "Ensure executable files are not writable", NULL }, { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Do not import extended attributes", NULL }, { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /)", "PATH" }, { "selinux-policy-from-base", 'P', 0, G_OPTION_ARG_NONE, &opt_selinux_policy_from_base, "Set SELinux labels based on first --tree argument", NULL }, @@ -194,13 +196,19 @@ commit_filter (OstreeRepo *self, g_file_info_set_attribute_uint32 (file_info, "unix::uid", opt_owner_uid); if (opt_owner_gid >= 0) g_file_info_set_attribute_uint32 (file_info, "unix::gid", opt_owner_gid); + guint mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); + + if (S_ISREG (mode) && opt_ro_executables && (mode & (S_IXUSR | S_IXGRP | S_IXOTH))) + { + mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); + g_file_info_set_attribute_uint32 (file_info, "unix::mode", mode); + } if (mode_adds && g_hash_table_lookup_extended (mode_adds, path, NULL, &value)) { - guint current_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode"); guint mode_add = GPOINTER_TO_UINT (value); g_file_info_set_attribute_uint32 (file_info, "unix::mode", - current_mode | mode_add); + mode | mode_add); g_hash_table_remove (mode_adds, path); } else if (mode_overrides && g_hash_table_lookup_extended (mode_overrides, path, NULL, &value)) @@ -572,6 +580,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio || opt_statoverride_file != NULL || opt_skiplist_file != NULL || opt_no_xattrs + || opt_ro_executables || opt_selinux_policy || opt_selinux_policy_from_base) { diff --git a/tests/basic-test.sh b/tests/basic-test.sh index ba88cc73..97cd05e2 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((88 + ${extra_basic_tests:-0}))" +echo "1..$((89 + ${extra_basic_tests:-0}))" CHECKOUT_U_ARG="" CHECKOUT_H_ARGS="-H" @@ -541,6 +541,24 @@ fi assert_file_has_mode checkout-test2-override/a/readable-only 600 echo "ok commit statoverride" +cd ${test_tmpdir} +rm test2-checkout -rf +$OSTREE checkout test2 test2-checkout +cd test2-checkout +install -m 0755 /dev/null user-wx +install -m 0575 /dev/null group-wx +install -m 0775 /dev/null both-wx +install -m 0555 /dev/null ugox +install -m 0644 /dev/null user-writable +cd .. +$OSTREE commit ${COMMIT_ARGS} -b test2-w-xor-x --mode-ro-executables --tree=dir=test2-checkout +$OSTREE ls test2-w-xor-x > ls.txt +for x in /{user,group,both}-wx; do + assert_file_has_content ls.txt '^-00555 .*'$x +done +assert_file_has_content ls.txt '^-00644 .*/user-writable' +echo "ok commit --mode-ro-executables" + cd ${test_tmpdir} cat > test-skiplist.txt < Date: Fri, 8 May 2020 12:49:55 +0000 Subject: [PATCH 126/177] ostree-prepare-root: Requires=sysroot.mount With just `After=` we'll still try to run in the scenario where `sysroot.mount` fails because the rootfs didn't appear. And this will end up spewing an error which can confuse people into thinking something is wrong at the ostree level. This has come up numerous times w/{Fedora,RHEL} CoreOS, most recently while looking at https://bugzilla.redhat.com/show_bug.cgi?id=1803130 --- src/boot/ostree-prepare-root.service | 1 + 1 file changed, 1 insertion(+) diff --git a/src/boot/ostree-prepare-root.service b/src/boot/ostree-prepare-root.service index 63357581..91692205 100644 --- a/src/boot/ostree-prepare-root.service +++ b/src/boot/ostree-prepare-root.service @@ -23,6 +23,7 @@ ConditionKernelCommandLine=ostree ConditionPathExists=/etc/initrd-release OnFailure=emergency.target After=sysroot.mount +Requires=sysroot.mount Before=initrd-root-fs.target [Service] From 588f42e8c64183dfa1fbaa08cc92c46b691b23c4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 7 May 2020 19:43:10 +0000 Subject: [PATCH 127/177] remote-add: Add --sign-verify=KEYTYPE=[inline|file]:PUBKEYREF Per https://github.com/ostreedev/ostree/issues/2080#issuecomment-623614483 A huge benefit of ed25519 (and ECC in general) is that keys are very short - short enough that it's completely reasonable to inline them into a command line argument. And I think that's a good model; it makes the keys very visible. For example, someone could easily copy-paste a commandline argument from a webpage (secured via TLS) that says to run `ostree remote add --sign-verify=ed25519=inline:KEY`. --- src/ostree/ot-remote-builtin-add.c | 57 ++++++++++++++++++++++++++++-- tests/test-signed-pull.sh | 27 ++++++++++++-- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index e4634710..a885336a 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -32,6 +32,7 @@ static gboolean opt_no_sign_verify; static gboolean opt_if_not_exists; static gboolean opt_force; static char *opt_gpg_import; +static char **opt_sign_verify; static char *opt_contenturl; static char *opt_collection_id; static char *opt_sysroot; @@ -46,6 +47,7 @@ static GOptionEntry option_entries[] = { { "set", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_set, "Set config option KEY=VALUE for remote", "KEY=VALUE" }, { "no-gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_no_gpg_verify, "Disable GPG verification", NULL }, { "no-sign-verify", 0, 0, G_OPTION_ARG_NONE, &opt_no_sign_verify, "Disable signature verification", NULL }, + { "sign-verify", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_sign_verify, "Verify signatures using KEYTYPE=inline:PUBKEY or KEYTYPE=file:/path/to/key", "KEYTYPE=[inline|file]:PUBKEY" }, { "if-not-exists", 0, 0, G_OPTION_ARG_NONE, &opt_if_not_exists, "Do nothing if the provided remote exists", NULL }, { "force", 0, 0, G_OPTION_ARG_NONE, &opt_force, "Replace the provided remote if it exists", NULL }, { "gpg-import", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_import, "Import GPG key from FILE", "FILE" }, @@ -57,6 +59,42 @@ static GOptionEntry option_entries[] = { { NULL } }; +static gboolean +add_verify_opt (GVariantBuilder *builder, + const char *keyspec, + GError **error) +{ + g_auto(GStrv) parts = g_strsplit (keyspec, "=", 2); + g_assert (parts && *parts); + const char *keytype = parts[0]; + if (!parts[1]) + return glnx_throw (error, "Failed to parse KEYTYPE=[inline|file]:DATA in %s", keyspec); + + g_autoptr(OstreeSign) sign = ostree_sign_get_by_name (keytype, error); + if (!sign) + return FALSE; + + const char *rest = parts[1]; + g_assert (!parts[2]); + g_auto(GStrv) keyparts = g_strsplit (rest, ":", 2); + g_assert (keyparts && *keyparts); + const char *keyref = keyparts[0]; + g_assert (keyref); + g_autofree char *optname = NULL; + if (g_str_equal (keyref, "inline")) + optname = g_strdup_printf ("verification-%s-key", keytype); + else if (g_str_equal (keyref, "file")) + optname = g_strdup_printf ("verification-%s-file", keytype); + else + return glnx_throw (error, "Invalid key reference %s, expected inline|file", keyref); + + g_assert (keyparts[1] && !keyparts[2]); + g_variant_builder_add (builder, "{s@v}", + optname, + g_variant_new_variant (g_variant_new_string (keyparts[1]))); + return TRUE; +} + gboolean ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { @@ -144,9 +182,24 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio #endif /* OSTREE_DISABLE_GPGME */ if (opt_no_sign_verify) + { + if (opt_sign_verify) + return glnx_throw (error, "Cannot specify both --sign-verify and --no-sign-verify"); + g_variant_builder_add (optbuilder, "{s@v}", + "sign-verify", + g_variant_new_variant (g_variant_new_boolean (FALSE))); + } + + for (char **iter = opt_sign_verify; iter && *iter; iter++) + { + const char *keyspec = *iter; + if (!add_verify_opt (optbuilder, keyspec, error)) + return FALSE; + } + if (opt_sign_verify) g_variant_builder_add (optbuilder, "{s@v}", - "sign-verify", - g_variant_new_variant (g_variant_new_boolean (FALSE))); + "sign-verify", + g_variant_new_variant (g_variant_new_boolean (TRUE))); if (opt_collection_id != NULL) g_variant_builder_add (optbuilder, "{s@v}", "collection-id", diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index 075c5f2b..b207eac2 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..11" +echo "1..16" # This is explicitly opt in for testing export OSTREE_DUMMY_SIGN_ENABLED=1 @@ -54,7 +54,7 @@ function test_signed_pull() { localsig=repo/$objpath mv $remotesig $remotesig.bak if ${CMD_PREFIX} ostree --repo=repo --depth=0 pull origin main; then - assert_not_reached "pull with sign-verify unexpectedly succeeded?" + assert_not_reached "pull with sign-verify and no commitmeta unexpectedly succeeded?" fi # ok now check that we can pull correctly mv $remotesig.bak $remotesig @@ -98,6 +98,25 @@ ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy ${CMD_PREFIX} ostree --repo=repo config unset 'remote "origin"'.verification-dummy-file test_signed_pull "dummy" "" +# Another test with the --verify option directly +repo_init --sign-verify=dummy=inline:${DUMMYSIGN} +test_signed_pull "dummy" "from remote opt" + +repo_init +if ${CMD_PREFIX} ostree --repo=repo remote add other --sign-verify=trustme=inline:ok http://localhost 2>err.txt; then + assert_not_reached "remote add with invalid keytype succeeded" +fi +assert_file_has_content err.txt 'Requested signature type is not implemented' +if ${CMD_PREFIX} ostree --repo=repo remote add other --sign-verify=dummy http://localhost 2>err.txt; then + assert_not_reached "remote add with invalid keytype succeeded" +fi +assert_file_has_content err.txt 'Failed to parse KEYTYPE' +if ${CMD_PREFIX} ostree --repo=repo remote add other --sign-verify=dummy=foo:bar http://localhost 2>err.txt; then + assert_not_reached "remote add with invalid keytype succeeded" +fi +assert_file_has_content err.txt 'Invalid key reference' +echo "ok remote add errs" + if ! has_sign_ed25519; then echo "ok ed25519-key pull signed commit # SKIP due libsodium unavailability" echo "ok ed25519-key re-pull signature for stored commit # SKIP due libsodium unavailability" @@ -105,6 +124,8 @@ if ! has_sign_ed25519; then echo "ok ed25519-key+file re-pull signature for stored commit # SKIP due libsodium unavailability" echo "ok ed25519-file pull signed commit # SKIP due libsodium unavailability" echo "ok ed25519-file re-pull signature for stored commit # SKIP due libsodium unavailability" + echo "ok ed25519-inline # SKIP due libsodium unavailability" + echo "ok ed25519-inline # SKIP due libsodium unavailability" exit 0 fi @@ -140,3 +161,5 @@ repo_init --set=sign-verify=true ${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-ed25519-file "${PUBKEYS}" test_signed_pull "ed25519" "file" +repo_init --sign-verify=ed25519=inline:"${ED25519PUBLIC}" +test_signed_pull "ed25519" "--verify-ed25519" From a9a81f3a297c08f027ccf5164c74f9ffdd0ca443 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 10 May 2020 13:20:50 +0000 Subject: [PATCH 128/177] signing: Change API to create instances directly This cleans up the verification code; it was weird how we'd get the list of known names and then try to create an instance from it (and throw an error if that failed, which couldn't happen). --- apidoc/ostree-sections.txt | 2 +- src/libostree/libostree-devel.sym | 2 +- src/libostree/ostree-repo-pull-verify.c | 37 ++++++++----------------- src/libostree/ostree-repo-pull.c | 11 +++----- src/libostree/ostree-sign.c | 37 +++++++++++-------------- src/libostree/ostree-sign.h | 2 +- 6 files changed, 34 insertions(+), 57 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 5586262a..979c8e93 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -709,7 +709,7 @@ ostree_kernel_args_to_string
ostree-sign OstreeSign -ostree_sign_list_names +ostree_sign_get_all ostree_sign_commit ostree_sign_commit_verify ostree_sign_data diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 9c6157a8..fd0ffd57 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -23,7 +23,7 @@ global: ostree_repo_commit_modifier_set_sepolicy_from_commit; someostree_symbol_deleteme; ostree_sign_get_type; - ostree_sign_list_names; + ostree_sign_get_all; ostree_sign_commit; ostree_sign_commit_verify; ostree_sign_data; diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c index c1eab6c3..84f7623b 100644 --- a/src/libostree/ostree-repo-pull-verify.c +++ b/src/libostree/ostree-repo-pull-verify.c @@ -142,6 +142,9 @@ _signapi_load_public_keys (OstreeSign *sign, return TRUE; } +/* Iterate over all known signing types, and check if the commit is signed + * by at least one. + */ gboolean _sign_verify_for_remote (OstreeRepo *repo, const gchar *remote_name, @@ -149,32 +152,18 @@ _sign_verify_for_remote (OstreeRepo *repo, GVariant *metadata, GError **error) { - /* list all signature types in detached metadata and check if signed by any? */ - g_auto (GStrv) names = ostree_sign_list_names(); guint n_invalid_signatures = 0; - guint n_unknown_signatures = 0; g_autoptr (GError) last_sig_error = NULL; gboolean found_sig = FALSE; - for (char **iter=names; iter && *iter; iter++) + g_autoptr(GPtrArray) signers = ostree_sign_get_all (); + for (guint i = 0; i < signers->len; i++) { - g_autoptr (OstreeSign) sign = NULL; - g_autoptr (GVariant) signatures = NULL; - const gchar *signature_key = NULL; - GVariantType *signature_format = NULL; - - if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) - { - n_unknown_signatures++; - continue; - } - - signature_key = ostree_sign_metadata_key (sign); - signature_format = (GVariantType *) ostree_sign_metadata_format (sign); - - signatures = g_variant_lookup_value (metadata, - signature_key, - signature_format); + OstreeSign *sign = signers->pdata[i]; + const gchar *signature_key = ostree_sign_metadata_key (sign); + GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format (sign); + g_autoptr (GVariant) signatures = + g_variant_lookup_value (metadata, signature_key, signature_format); /* If not found signatures for requested signature subsystem */ if (!signatures) @@ -201,11 +190,7 @@ _sign_verify_for_remote (OstreeRepo *repo, } if (!found_sig) - { - if (n_unknown_signatures > 0) - return glnx_throw (error, "No signatures found (%d unknown type)", n_unknown_signatures); - return glnx_throw (error, "No signatures found"); - } + return glnx_throw (error, "No signatures found"); g_assert (last_sig_error); g_propagate_error (error, g_steal_pointer (&last_sig_error)); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 817307e9..4d617c96 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1544,14 +1544,11 @@ scan_commit_object (OtPullData *pull_data, gboolean found_any_signature = FALSE; gboolean found_valid_signature = FALSE; - /* list all signature types in detached metadata and check if signed by any? */ - g_auto (GStrv) names = ostree_sign_list_names(); - for (char **iter=names; iter && *iter; iter++) + /* FIXME - dedup this with _sign_verify_for_remote() */ + g_autoptr(GPtrArray) signers = ostree_sign_get_all (); + for (guint i = 0; i < signers->len; i++) { - g_autoptr (OstreeSign) sign = NULL; - - if ((sign = ostree_sign_get_by_name (*iter, NULL)) == NULL) - continue; + OstreeSign *sign = signers->pdata[i]; /* Try to load public key(s) according remote's configuration */ if (!_signapi_load_public_keys (sign, pull_data->repo, pull_data->remote_name, error)) diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index 68447da6..f3992480 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -436,8 +436,6 @@ ostree_sign_commit_verify (OstreeSign *self, * * Return the pointer to the name of currently used/selected signing engine. * - * The list of available engines could be acquired with #ostree_sign_list_names. - * * Returns: (transfer none): pointer to the name * @NULL in case of error (unlikely). * @@ -515,28 +513,27 @@ ostree_sign_commit (OstreeSign *self, } /** - * ostree_sign_list_names: + * ostree_sign_get_all: * - * Return an array with all available sign engines names. + * Return an array with newly allocated instances of all available + * signing engines; they will not be initialized. * - * Returns: (transfer full): an array of strings, free when you used it + * Returns: (transfer full) (element-type OstreeSign): an array of signing engines * * Since: 2020.2 */ -GStrv -ostree_sign_list_names(void) +GPtrArray * +ostree_sign_get_all (void) { + g_autoptr(GPtrArray) engines = g_ptr_array_new_with_free_func (g_object_unref); + for (guint i = 0; i < G_N_ELEMENTS(sign_types); i++) + { + OstreeSign *engine = ostree_sign_get_by_name (sign_types[i].name, NULL); + g_assert (engine); + g_ptr_array_add (engines, engine); + } - GStrv names = g_new0 (char *, G_N_ELEMENTS(sign_types) + 1); - gint i = 0; - - for (i=0; i < G_N_ELEMENTS(sign_types); i++) - { - names[i] = g_strdup(sign_types[i].name); - g_debug ("Found '%s' signing engine", names[i]); - } - - return names; + return g_steal_pointer (&engines); } /** @@ -544,11 +541,9 @@ ostree_sign_list_names(void) * @name: the name of desired signature engine * @error: return location for a #GError * - * Tries to find and return proper signing engine by it's name. + * Create a new instance of a signing engine. * - * The list of available engines could be acquired with #ostree_sign_list_names. - * - * Returns: (transfer full): a constant, free when you used it + * Returns: (transfer full): New signing engine, or %NULL if the engine is not known * * Since: 2020.2 */ diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index da10469b..588ace53 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -153,7 +153,7 @@ gboolean ostree_sign_load_pk (OstreeSign *self, _OSTREE_PUBLIC -GStrv ostree_sign_list_names(void); +GPtrArray * ostree_sign_get_all(void); _OSTREE_PUBLIC OstreeSign * ostree_sign_get_by_name (const gchar *name, GError **error); From 474e1e2a3372186b843b613d5b0be5fcd86547de Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 11 May 2020 21:21:54 +0000 Subject: [PATCH 129/177] tests/staged-delay.sh: New test Attempting to reproduce https://bugzilla.redhat.com/show_bug.cgi?id=1827712 but no dice yet. --- tests/kola/destructive/staged-delay.sh | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 tests/kola/destructive/staged-delay.sh diff --git a/tests/kola/destructive/staged-delay.sh b/tests/kola/destructive/staged-delay.sh new file mode 100755 index 00000000..a320202f --- /dev/null +++ b/tests/kola/destructive/staged-delay.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -xeuo pipefail + +# Add an artificial delay into ostree-finalize-staged.service +# and verify it sees /boot; https://bugzilla.redhat.com/show_bug.cgi?id=1827712 + +. ${KOLA_EXT_DATA}/libinsttest.sh + +require_writable_sysroot +prepare_tmpdir + +n=$(nth_boot) +case "${n}" in +1) +dropin=/etc/systemd/system/ostree-finalize-staged.service.d/delay.conf +mkdir -p $(dirname ${dropin}) +cat >"${dropin}" << 'EOF' +[Service] +ExecStop=/bin/sh -c 'sleep 10 && if ! test -d /boot/loader/entries; then echo error: no /boot/loader/entries; exit 1; fi; echo ostree-finalize-staged found /boot/loader/entries' +#ExecStop=/bin/false +EOF +systemctl daemon-reload +rpm-ostree kargs --append=somedummykarg=1 +kola_reboot +;; + +2) +journalctl -b -1 -u ostree-finalize-staged > logs.txt +assert_file_has_content_literal logs.txt 'ostree-finalize-staged found /boot/loader/entries' +# older systemd doesn't output the success message +if systemctl --version | head -1 | grep -qF -e 'systemd 239'; then + assert_file_has_content_literal logs.txt 'Stopped OSTree Finalize Staged Deployment' + assert_not_file_has_content logs.txt 'Failed with result' +else + assert_file_has_content_literal logs.txt 'ostree-finalize-staged.service: Succeeded.' +fi +assert_file_has_content_literal /proc/cmdline somedummykarg=1 +;; +esac +echo ok From 9509a4bc948672b6bdf3ccdd0ea882a141c78974 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 12 May 2020 01:26:00 +0000 Subject: [PATCH 130/177] pull: Further cleanup signapi verification Previously in the pull code, every time we went to verify a commit we would re-initialize an `OstreeSign` instance of each time, re-parse the remote configuration and re-load its public keys etc. In most cases this doesn't matter really because we're pulling one commit, but if e.g. pulling a commit with history would get a bit silly. This changes things so that the pull code initializes the verifiers once, and reuses them thereafter. This is continuing towards changing the code to support explicitly configured verifiers, xref https://github.com/ostreedev/ostree/issues/2080 --- src/libostree/ostree-repo-pull-private.h | 20 ++++++------ src/libostree/ostree-repo-pull-verify.c | 41 ++++++++++++++++-------- src/libostree/ostree-repo-pull.c | 32 +++++++++++------- 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index b9eb2342..5bc2a33a 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -123,6 +123,8 @@ typedef struct { gboolean is_commit_only; OstreeRepoImportFlags importflags; + GPtrArray *signapi_verifiers; + GPtrArray *dirs; gboolean have_previous_bytes; @@ -137,18 +139,16 @@ typedef struct { GSource *idle_src; } OtPullData; -gboolean -_sign_verify_for_remote (OstreeRepo *repo, - const gchar *remote_name, - GBytes *signed_data, - GVariant *metadata, - GError **error); +GPtrArray * +_signapi_verifiers_for_remote (OstreeRepo *repo, + const char *remote_name, + GError **error); gboolean -_signapi_load_public_keys (OstreeSign *sign, - OstreeRepo *repo, - const gchar *remote_name, - GError **error); +_sign_verify_for_remote (GPtrArray *signers, + GBytes *signed_data, + GVariant *metadata, + GError **error); gboolean _verify_unwritten_commit (OtPullData *pull_data, diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c index 84f7623b..36d877ac 100644 --- a/src/libostree/ostree-repo-pull-verify.c +++ b/src/libostree/ostree-repo-pull-verify.c @@ -67,7 +67,7 @@ get_signapi_remote_option (OstreeRepo *repo, * Returns: %FALSE if any source is configured but nothing has been loaded. * Returns: %TRUE if no configuration or any key loaded. * */ -gboolean +static gboolean _signapi_load_public_keys (OstreeSign *sign, OstreeRepo *repo, const gchar *remote_name, @@ -142,21 +142,40 @@ _signapi_load_public_keys (OstreeSign *sign, return TRUE; } -/* Iterate over all known signing types, and check if the commit is signed +/* Create a new array of OstreeSign objects and load the public + * keys as described by the remote configuration. + */ +GPtrArray * +_signapi_verifiers_for_remote (OstreeRepo *repo, + const char *remote_name, + GError **error) +{ + g_autoptr(GPtrArray) signers = ostree_sign_get_all (); + g_assert_cmpuint (signers->len, >=, 1); + for (guint i = 0; i < signers->len; i++) + { + OstreeSign *sign = signers->pdata[i]; + /* Try to load public key(s) according remote's configuration */ + if (!_signapi_load_public_keys (sign, repo, remote_name, error)) + return FALSE; + } + return g_steal_pointer (&signers); +} + +/* Iterate over the configured signers, and require the commit is signed * by at least one. */ gboolean -_sign_verify_for_remote (OstreeRepo *repo, - const gchar *remote_name, - GBytes *signed_data, - GVariant *metadata, - GError **error) +_sign_verify_for_remote (GPtrArray *signers, + GBytes *signed_data, + GVariant *metadata, + GError **error) { guint n_invalid_signatures = 0; g_autoptr (GError) last_sig_error = NULL; gboolean found_sig = FALSE; - g_autoptr(GPtrArray) signers = ostree_sign_get_all (); + g_assert_cmpuint (signers->len, >=, 1); for (guint i = 0; i < signers->len; i++) { OstreeSign *sign = signers->pdata[i]; @@ -169,10 +188,6 @@ _sign_verify_for_remote (OstreeRepo *repo, if (!signatures) continue; - /* Try to load public key(s) according remote's configuration */ - if (!_signapi_load_public_keys (sign, repo, remote_name, error)) - return FALSE; - found_sig = TRUE; /* Return true if any signature fit to pre-loaded public keys. @@ -275,7 +290,7 @@ _verify_unwritten_commit (OtPullData *pull_data, if (detached_metadata == NULL) return glnx_throw (error, "Can't verify commit without detached metadata"); - if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, signed_data, detached_metadata, error)) + if (!_sign_verify_for_remote (pull_data->signapi_verifiers, signed_data, detached_metadata, error)) return glnx_prefix_error (error, "Can't verify commit"); /* Mark the commit as verified to avoid double verification diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4d617c96..363db9e2 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1544,15 +1544,10 @@ scan_commit_object (OtPullData *pull_data, gboolean found_any_signature = FALSE; gboolean found_valid_signature = FALSE; - /* FIXME - dedup this with _sign_verify_for_remote() */ - g_autoptr(GPtrArray) signers = ostree_sign_get_all (); - for (guint i = 0; i < signers->len; i++) + g_assert (pull_data->signapi_verifiers); + for (guint i = 0; i < pull_data->signapi_verifiers->len; i++) { - OstreeSign *sign = signers->pdata[i]; - - /* Try to load public key(s) according remote's configuration */ - if (!_signapi_load_public_keys (sign, pull_data->repo, pull_data->remote_name, error)) - return FALSE; + OstreeSign *sign = pull_data->signapi_verifiers->pdata[i]; found_any_signature = TRUE; @@ -3574,6 +3569,15 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } + if (pull_data->sign_verify || pull_data->sign_verify_summary) + { + g_assert (pull_data->remote_name != NULL); + pull_data->signapi_verifiers = _signapi_verifiers_for_remote (pull_data->repo, pull_data->remote_name, error); + if (!pull_data->signapi_verifiers) + goto out; + g_assert_cmpint (pull_data->signapi_verifiers->len, >=, 1); + } + pull_data->phase = OSTREE_PULL_PHASE_FETCHING_REFS; if (!reinitialize_fetcher (pull_data, remote_name_or_baseurl, error)) @@ -3954,7 +3958,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, bytes_sig, FALSE); - if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, &temp_error)) + g_assert (pull_data->signapi_verifiers); + if (!_sign_verify_for_remote (pull_data->signapi_verifiers, bytes_summary, signatures, &temp_error)) { if (summary_from_cache) { @@ -3983,7 +3988,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (!_sign_verify_for_remote (pull_data->repo, pull_data->remote_name, bytes_summary, signatures, error)) + if (!_sign_verify_for_remote (pull_data->signapi_verifiers, bytes_summary, signatures, error)) goto out; } else @@ -4586,6 +4591,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_free (pull_data->remote_refspec_name); g_free (pull_data->remote_name); g_free (pull_data->append_user_agent); + g_clear_pointer (&pull_data->signapi_verifiers, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref); @@ -6089,8 +6095,10 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, signatures, FALSE); - - if (!_sign_verify_for_remote (self, name, summary, sig_variant, error)) + g_autoptr(GPtrArray) signapi_verifiers = _signapi_verifiers_for_remote (self, name, error); + if (!signapi_verifiers) + goto out; + if (!_sign_verify_for_remote (signapi_verifiers, summary, sig_variant, error)) goto out; } } From 82679ce834732ba4941b097cd83c43902c0d6092 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 12 May 2020 22:04:53 +0000 Subject: [PATCH 131/177] finalize: Add RequiresMountsFor=/boot too In https://bugzilla.redhat.com/show_bug.cgi?id=1827712 some OpenShift CI is seeing `/boot` being unmounted before `ostree-finalize-staged.service` runs or completes. We finally tracked this down to a bug elsewhere, but I think we should add this because it clearly shows our requirements. --- src/boot/ostree-finalize-staged.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boot/ostree-finalize-staged.service b/src/boot/ostree-finalize-staged.service index 8152e596..90e212b0 100644 --- a/src/boot/ostree-finalize-staged.service +++ b/src/boot/ostree-finalize-staged.service @@ -23,7 +23,7 @@ Documentation=man:ostree(1) ConditionPathExists=/run/ostree-booted DefaultDependencies=no -RequiresMountsFor=/sysroot +RequiresMountsFor=/sysroot /boot After=local-fs.target Before=basic.target final.target # We want to make sure the transaction logs are persisted to disk: From c8efce06564b7adef83994dddb41cd61a030207d Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 14 May 2020 13:44:32 -0400 Subject: [PATCH 132/177] lib/pull: Add `timestamp-check-from-rev` The way `timestamp-check` works might be too restrictive in some situations. Essentially, we need to support the case where users want to pull an older commit than the current tip, but while still guaranteeing that it is newer than some even older commit. This will be used in Fedora CoreOS. For more information see: https://github.com/coreos/rpm-ostree/pull/2094 https://github.com/coreos/fedora-coreos-tracker/issues/481 --- src/libostree/ostree-repo-pull-private.h | 1 + src/libostree/ostree-repo-pull.c | 29 +++++++++++++++++++++++- src/ostree/ot-builtin-pull.c | 5 ++++ tests/pull-test.sh | 24 +++++++++++++++++++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index 5bc2a33a..86d1ffee 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -114,6 +114,7 @@ typedef struct { guint n_imported_content; gboolean timestamp_check; /* Verify commit timestamps */ + char *timestamp_check_from_rev; int maxdepth; guint64 max_metadata_size; guint64 start_time; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 363db9e2..291f3fe6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1593,6 +1593,7 @@ scan_commit_object (OtPullData *pull_data, commit, error)) return glnx_prefix_error (error, "Commit %s", checksum); + guint64 new_ts = ostree_commit_get_timestamp (commit); if (pull_data->timestamp_check) { /* We don't support timestamp checking while recursing right now */ @@ -1611,11 +1612,22 @@ scan_commit_object (OtPullData *pull_data, return glnx_prefix_error (error, "Reading %s for timestamp-check", ref->ref_name); guint64 orig_ts = ostree_commit_get_timestamp (orig_commit); - guint64 new_ts = ostree_commit_get_timestamp (commit); if (!_ostree_compare_timestamps (orig_rev, orig_ts, checksum, new_ts, error)) return FALSE; } } + if (pull_data->timestamp_check_from_rev) + { + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_commit (pull_data->repo, pull_data->timestamp_check_from_rev, + &commit, NULL, error)) + return glnx_prefix_error (error, "Reading %s for timestamp-check-from-rev", + pull_data->timestamp_check_from_rev); + + guint64 ts = ostree_commit_get_timestamp (commit); + if (!_ostree_compare_timestamps (pull_data->timestamp_check_from_rev, ts, checksum, new_ts, error)) + return FALSE; + } /* If we found a legacy transaction flag, assume all commits are partial */ gboolean is_partial = commitstate_is_partial (pull_data, commitstate); @@ -3270,6 +3282,7 @@ initiate_request (OtPullData *pull_data, * * require-static-deltas (b): Require static deltas * * override-commit-ids (as): Array of specific commit IDs to fetch for refs * * timestamp-check (b): Verify commit timestamps are newer than current (when pulling via ref); Since: 2017.11 + * * timestamp-check-from-rev (s): Verify that all fetched commit timestamps are newer than timestamp of given rev; Since: 2020.4 * * metadata-size-restriction (t): Restrict metadata objects to a maximum number of bytes; 0 to disable. Since: 2018.9 * * dry-run (b): Only print information on what will be downloaded (requires static deltas) * * override-url (s): Fetch objects from this URL if remote specifies no metalink in options @@ -3372,6 +3385,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "update-frequency", "u", &update_frequency); (void) g_variant_lookup (options, "localcache-repos", "^a&s", &opt_localcache_repos); (void) g_variant_lookup (options, "timestamp-check", "b", &pull_data->timestamp_check); + (void) g_variant_lookup (options, "timestamp-check-from-rev", "s", &pull_data->timestamp_check_from_rev); (void) g_variant_lookup (options, "max-metadata-size", "t", &pull_data->max_metadata_size); (void) g_variant_lookup (options, "append-user-agent", "s", &pull_data->append_user_agent); opt_n_network_retries_set = @@ -4259,6 +4273,18 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_steal_pointer (&checksum)); } + /* Resolve refs to a checksum if necessary */ + if (pull_data->timestamp_check_from_rev && + !ostree_validate_checksum_string (pull_data->timestamp_check_from_rev, NULL)) + { + g_autofree char *from_rev = NULL; + if (!ostree_repo_resolve_rev (pull_data->repo, pull_data->timestamp_check_from_rev, FALSE, + &from_rev, error)) + goto out; + g_free (pull_data->timestamp_check_from_rev); + pull_data->timestamp_check_from_rev = g_steal_pointer (&from_rev); + } + g_hash_table_unref (requested_refs_to_fetch); requested_refs_to_fetch = g_steal_pointer (&updated_requested_refs_to_fetch); if (g_hash_table_size (requested_refs_to_fetch) == 1) @@ -4604,6 +4630,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->fetched_detached_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->ref_original_commits, (GDestroyNotify) g_hash_table_unref); + g_free (pull_data->timestamp_check_from_rev); g_clear_pointer (&pull_data->verified_commits, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->ref_keyring_map, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 1fae0a38..1625ab47 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -37,6 +37,7 @@ static gboolean opt_require_static_deltas; static gboolean opt_untrusted; static gboolean opt_http_trusted; static gboolean opt_timestamp_check; +static char* opt_timestamp_check_from_rev; static gboolean opt_bareuseronly_files; static char** opt_subpaths; static char** opt_http_headers; @@ -72,6 +73,7 @@ static GOptionEntry options[] = { { "network-retries", 0, 0, G_OPTION_ARG_INT, &opt_network_retries, "Specifies how many times each download should be retried upon error (default: 5)", "N"}, { "localcache-repo", 'L', 0, G_OPTION_ARG_FILENAME_ARRAY, &opt_localcache_repos, "Add REPO as local cache source for objects during this pull", "REPO" }, { "timestamp-check", 'T', 0, G_OPTION_ARG_NONE, &opt_timestamp_check, "Require fetched commits to have newer timestamps", NULL }, + { "timestamp-check-from-rev", 0, 0, G_OPTION_ARG_STRING, &opt_timestamp_check_from_rev, "Require fetched commits to have newer timestamps than given rev", NULL }, /* let's leave this hidden for now; we just need it for tests */ { "append-user-agent", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &opt_append_user_agent, "Append string to user agent", NULL }, { NULL } @@ -313,6 +315,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, if (opt_timestamp_check) g_variant_builder_add (&builder, "{s@v}", "timestamp-check", g_variant_new_variant (g_variant_new_boolean (opt_timestamp_check))); + if (opt_timestamp_check_from_rev) + g_variant_builder_add (&builder, "{s@v}", "timestamp-check-from-rev", + g_variant_new_variant (g_variant_new_string (opt_timestamp_check_from_rev))); if (override_commit_ids) g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", diff --git a/tests/pull-test.sh b/tests/pull-test.sh index a52adab9..92bcf112 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -318,7 +318,7 @@ ${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt assert_file_has_content main.txt ${rev} echo "ok pull specific commit" -# test pull -T +# test pull -T and --timestamp-check-from-rev cd ${test_tmpdir} repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main @@ -347,6 +347,28 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older" assert_streq ${newrev} "$(${CMD_PREFIX} ostree --repo=repo rev-parse main)" # But we can pull it without timestamp checking ${CMD_PREFIX} ostree --repo=repo pull origin main +# Now test --timestamp-check-from-rev. First, add two new commits with distinct +# but newer timestamps. +oldrev=${newrev2} +middlerev=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main --tree=ref=main) +sleep 1 +latestrev=$(${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo commit ${COMMIT_ARGS} -b main --tree=ref=main) +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u +# OK, let's pull the latest now. +${CMD_PREFIX} ostree --repo=repo pull -T origin main +assert_streq ${latestrev} "$(${CMD_PREFIX} ostree --repo=repo rev-parse main)" +# Check we can't pull the middle commit by overrides with ts checking on +if ${CMD_PREFIX} ostree --repo=repo pull -T origin main@${middlerev} 2>err.txt; then + fatal "pulled older commit override with timestamp checking enabled?" +fi +assert_file_has_content err.txt "Upgrade.*is chronologically older" +# Check we can't pull an older commit by override if it's newer than --timestamp-check-from-rev +if ${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${latestrev} origin main@${middlerev} 2>err.txt; then + fatal "pulled older commit override with timestamp checking enabled?" +fi +assert_file_has_content err.txt "Upgrade.*is chronologically older" +# But we can pull it with --timestamp-check-from-rev when starting from the oldrev +${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev} echo "ok pull timestamp checking" cd ${test_tmpdir} From 79079c265759613e254b377a937c1baa34cc109e Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 14 May 2020 15:05:45 -0400 Subject: [PATCH 133/177] lib/upgrader: Pull with `timestamp-check-from-rev` For the same reason as https://github.com/coreos/rpm-ostree/pull/2094. What we care most about is that the new commit we pull is newer than the one we're currently sitting on, not necessarily that it's newer than the branch itself, which it might not be if e.g. we're trying to deploy a commit older than the tip but still newer than the deployment (via `--override-commit`). --- src/libostree/ostree-sysroot-upgrader.c | 4 ++-- tests/admin-test.sh | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-sysroot-upgrader.c b/src/libostree/ostree-sysroot-upgrader.c index 8fb231a3..46813358 100644 --- a/src/libostree/ostree-sysroot-upgrader.c +++ b/src/libostree/ostree-sysroot-upgrader.c @@ -530,8 +530,8 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self, g_variant_new_variant (g_variant_new_int32 (flags))); /* Add the timestamp check, unless disabled */ if ((upgrader_flags & OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER) == 0) - g_variant_builder_add (optbuilder, "{s@v}", "timestamp-check", - g_variant_new_variant (g_variant_new_boolean (TRUE))); + g_variant_builder_add (optbuilder, "{s@v}", "timestamp-check-from-rev", + g_variant_new_variant (g_variant_new_string (from_revision))); g_variant_builder_add (optbuilder, "{s@v}", "refs", g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch, -1))); diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 11b9ea14..03b455a3 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((27 + ${extra_admin_tests:-0}))" +echo "1..$((28 + ${extra_admin_tests:-0}))" mkdir sysrootmin ${CMD_PREFIX} ostree admin init-fs --modern sysrootmin @@ -304,8 +304,14 @@ ${CMD_PREFIX} ostree pull --repo=sysroot/ostree/repo --commit-metadata-only --de head_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime^^^^) assert_not_streq ${head_rev} ${prev_rev} +# check that we can't "upgrade" to an older commit without --allow-downgrade +if ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} 2> err.txt; then + fatal "downgraded without --allow-downgrade?" +fi +assert_file_has_content err.txt "Upgrade.*is chronologically older" ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} --allow-downgrade curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) + assert_streq ${curr_rev} ${prev_rev} ${CMD_PREFIX} ostree admin upgrade --os=testos curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) @@ -313,6 +319,18 @@ assert_streq ${curr_rev} ${head_rev} echo "ok upgrade with and without override-commit" +# check that we can still upgrade to a rev that's not the tip of the branch but +# that's still newer than the deployment +sleep 1 +os_repository_new_commit +sleep 1 +os_repository_new_commit +${CMD_PREFIX} ostree pull --repo=sysroot/ostree/repo --commit-metadata-only --depth=-1 testos:testos/buildmaster/x86_64-runtime +curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) +prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime^) +${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} +echo "ok upgrade to newer version older than branch tip" + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" \ --add-metadata-string 'ostree.source-title=libtest os_repository_new_commit()' -b testos/buildmaster/x86_64-runtime \ -s "Build" --tree=dir=${test_tmpdir}/osdata From 48f5a1885e3c6a4db5bc47187eab3b0870a463d3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 14 May 2020 23:50:40 +0000 Subject: [PATCH 134/177] ci: Install kola tests This builds on https://github.com/coreos/coreos-assembler/pull/1441 to install our tests rather than running them from the source directory. This model will more cleanly allow us to ship our tests along with a test container or elsewhere, separate from the source directory. Also prep for https://github.com/ostreedev/ostree/pull/2048 --- .cci.jenkinsfile | 3 +++ Makefile-tests.am | 7 +++++++ tests/kola/Makefile | 13 +++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 tests/kola/Makefile diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index d1f2a231..68656cbd 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -76,6 +76,9 @@ parallel fcos: { rmdir insttree coreos-assembler fetch coreos-assembler build + # Install the tests, then be sure they're not run again + make -C tests/kola install + rm tests/kola -rf """) } fcosKola(cosaDir: "${env.WORKSPACE}") diff --git a/Makefile-tests.am b/Makefile-tests.am index a172f015..411c5628 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -456,3 +456,10 @@ else endif INSTALL_DATA_HOOKS += install-installed-tests-extra endif + +# Just forward these +build-kola-tests: + $(MAKE) -C tests/kola + +install-kola-tests: + $(MAKE) -C tests/kola install diff --git a/tests/kola/Makefile b/tests/kola/Makefile new file mode 100644 index 00000000..18305a2f --- /dev/null +++ b/tests/kola/Makefile @@ -0,0 +1,13 @@ +DESTDIR ?= + +TESTDIRS := $(shell find -mindepth 1 -maxdepth 1 -type d) +LIBSCRIPTS := $(shell ls *.sh) + +KOLA_TESTDIR = $(DESTDIR)/usr/lib/coreos-assembler/tests/kola/ostree/ + +all: + for x in $(LIBSCRIPTS); do bash -n "$${x}"; done + +install: + install -D -m 0644 -t $(KOLA_TESTDIR) $(LIBSCRIPTS) + for x in $(TESTDIRS); do rsync -rlv ./$${x} $(KOLA_TESTDIR)/; done From 5a47c926c1762e3dd4397a71b13e8f8a475c5079 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 17 May 2020 13:52:24 +0000 Subject: [PATCH 135/177] pull: Only have API to disable signapi for local pulls There's a lot of historical baggage associated with GPG verification and `ostree pull` versus `ostree pull-local`. In particular nowadays, if you use a `file://` remote things are transparently optimized to e.g. use reflinks if available. So for anyone who doesn't trust the "remote" repository, you should really go through through the regular `ostree remote add --sign-verify=X file://` path for example. Having a mechanism to say "turn on signapi verification" *without* providing keys goes back into the "global state" debate I brought up in https://github.com/ostreedev/ostree/issues/2080 It's just much cleaner architecturally if there is exactly one path to find keys: from a remote config. So here in contrast to the GPG code, for `pull-local` we explictily disable signapi validation, and the `ostree_repo_pull()` API just surfaces flags to disable it, not enable it. --- src/libostree/ostree-repo-pull.c | 52 ++++++++++++++++++------------ src/ostree/ot-builtin-pull-local.c | 18 ++++------- tests/test-local-pull.sh | 38 +--------------------- 3 files changed, 40 insertions(+), 68 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 291f3fe6..7116c3dc 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3277,6 +3277,8 @@ initiate_request (OtPullData *pull_data, * * override-remote-name (s): If local, add this remote to refspec * * gpg-verify (b): GPG verify commits * * gpg-verify-summary (b): GPG verify summary + * * disable-sign-verify (b): Disable signapi verification of commits + * * disable-sign-verify-summary (b): Disable signapi verification of the summary * * depth (i): How far in the history to traverse; default is 0, -1 means infinite * * disable-static-deltas (b): Do not use static deltas * * require-static-deltas (b): Require static deltas @@ -3334,11 +3336,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GSource) update_timeout = NULL; gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; - gboolean opt_sign_verify_set = FALSE; - gboolean opt_sign_verify_summary_set = FALSE; gboolean opt_collection_refs_set = FALSE; gboolean opt_n_network_retries_set = FALSE; gboolean opt_ref_keyring_map_set = FALSE; + gboolean disable_sign_verify = FALSE; + gboolean disable_sign_verify_summary = FALSE; const char *main_collection_id = NULL; const char *url_override = NULL; gboolean inherit_transaction = FALSE; @@ -3370,10 +3372,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_variant_lookup (options, "gpg-verify", "b", &pull_data->gpg_verify); opt_gpg_verify_summary_set = g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary); - opt_sign_verify_set = - g_variant_lookup (options, "sign-verify", "b", &pull_data->sign_verify); - opt_sign_verify_summary_set = - g_variant_lookup (options, "sign-verify-summary", "b", &pull_data->sign_verify_summary); + g_variant_lookup (options, "disable-sign-verify", "b", &disable_sign_verify); + g_variant_lookup (options, "disable-sign-verify-summary", "b", &disable_sign_verify_summary); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &pull_data->disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas); @@ -3525,8 +3525,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, * pulls by default. */ if ((pull_data->gpg_verify || - pull_data->gpg_verify_summary || - pull_data->sign_verify + pull_data->gpg_verify_summary ) && pull_data->remote_name == NULL) { @@ -3553,18 +3552,31 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, pull_data->remote_name, &pull_data->gpg_verify_summary, error)) goto out; - /* Fetch verification settings from remote if it wasn't already - * explicitly set in the options. */ - if (!opt_sign_verify_set) - if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, - "sign-verify", FALSE, - &pull_data->sign_verify, error)) - goto out; - if (!opt_sign_verify_summary_set) - if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, - "sign-verify-summary", FALSE, - &pull_data->sign_verify_summary, error)) - goto out; + /* signapi differs from GPG in that it can only be explicitly *disabled* + * transiently during pulls, not enabled. + */ + if (disable_sign_verify) + { + pull_data->sign_verify = FALSE; + } + else + { + if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, + "sign-verify", FALSE, + &pull_data->sign_verify, error)) + goto out; + } + if (disable_sign_verify_summary) + { + pull_data->sign_verify_summary = FALSE; + } + else + { + if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, + "sign-verify-summary", FALSE, + &pull_data->sign_verify_summary, error)) + goto out; + } /* NOTE: If changing this, see the matching implementation in * ostree-sysroot-upgrader.c diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index 695b09e5..c42d38d7 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -39,8 +39,6 @@ static gboolean opt_bareuseronly_files; static gboolean opt_require_static_deltas; static gboolean opt_gpg_verify; static gboolean opt_gpg_verify_summary; -static gboolean opt_sign_verify; -static gboolean opt_sign_verify_summary; static int opt_depth = 0; /* ATTENTION: @@ -57,8 +55,6 @@ static GOptionEntry options[] = { { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify, "GPG verify commits (must specify --remote)", NULL }, { "gpg-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify_summary, "GPG verify summary (must specify --remote)", NULL }, - { "sign-verify", 0, 0, G_OPTION_ARG_NONE, &opt_sign_verify, "Verify commits signature (must specify --remote)", NULL }, - { "sign-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_sign_verify, "Verify summary signature (must specify --remote)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -185,13 +181,13 @@ ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invoc g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "depth", g_variant_new_variant (g_variant_new_int32 (opt_depth))); - - if (opt_sign_verify) - g_variant_builder_add (&builder, "{s@v}", "sign-verify", - g_variant_new_variant (g_variant_new_boolean (TRUE))); - if (opt_sign_verify_summary) - g_variant_builder_add (&builder, "{s@v}", "sign-verify-summary", - g_variant_new_variant (g_variant_new_boolean (TRUE))); + /* local pulls always disable signapi verification. If you don't want this, use + * ostree remote add --sign-verify= file:// + */ + g_variant_builder_add (&builder, "{s@v}", "disable-sign-verify", + g_variant_new_variant (g_variant_new_boolean (TRUE))); + g_variant_builder_add (&builder, "{s@v}", "disable-sign-verify-summary", + g_variant_new_variant (g_variant_new_boolean (TRUE))); if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index d443421a..555e9b26 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -28,7 +28,7 @@ unset OSTREE_GPG_HOME skip_without_user_xattrs -echo "1..11" +echo "1..8" setup_test_repository "archive" echo "ok setup" @@ -115,39 +115,3 @@ for src_object in `find repo/objects -name '*.filez'`; do assert_files_hardlinked "$src_object" "$dst_object" done echo "ok pull-local z2 to z2 default hardlink" - -if has_sign_ed25519; then - gen_ed25519_keys - - mkdir repo8 - ostree_repo_init repo8 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo8 remote add --set=verification-ed25519-key="${ED25519PUBLIC}" origin repo - cat repo8/config - - if ${CMD_PREFIX} ostree --repo=repo8 pull-local --remote=origin --sign-verify repo test2 2>err.txt; then - assert_not_reached "Ed25519 signature verification unexpectedly succeeded" - fi - assert_file_has_content err.txt 'ed25519: commit have no signatures of my type' - echo "ok --sign-verify with no signature" - - ${OSTREE} sign test2 ${ED25519SECRET} - - mkdir repo9 - ostree_repo_init repo9 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo9 remote add --set=verification-ed25519-key="$(gen_ed25519_random_public)" origin repo - if ${CMD_PREFIX} ostree --repo=repo9 pull-local --remote=origin --sign-verify repo test2 2>err.txt; then - assert_not_reached "Ed25519 signature verification unexpectedly succeeded" - fi - assert_file_has_content err.txt 'no valid ed25519 signatures found' - echo "ok --sign-verify with wrong signature" - - mkdir repo10 - ostree_repo_init repo10 --mode="archive" - ${CMD_PREFIX} ostree --repo=repo10 remote add --set=verification-ed25519-key="${ED25519PUBLIC}" origin repo - ${CMD_PREFIX} ostree --repo=repo10 pull-local --remote=origin --sign-verify repo test2 - echo "ok --sign-verify" -else - echo "ok --sign-verify with no signature | # SKIP due libsodium unavailability" - echo "ok --sign-verify with wrong signature | # SKIP due libsodium unavailability" - echo "ok --sign-verify | # SKIP libsodium unavailability" -fi From f14aa894d36463cf13fbfa404526a2a57d45fcd6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 17 May 2020 14:27:45 +0000 Subject: [PATCH 136/177] ci: Test for clock skew I saw `tests/test-admin-deploy.none.sh` fail in one CI run, and I want to check if it was because of clock skew, so fail fast if we detect that. xref https://github.com/ostreedev/ostree/pull/2099#issuecomment-629805375 --- tests/admin-test.sh | 2 ++ tests/libtest.sh | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 03b455a3..c716d0a5 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -306,6 +306,8 @@ prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buil assert_not_streq ${head_rev} ${prev_rev} # check that we can't "upgrade" to an older commit without --allow-downgrade if ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} 2> err.txt; then + ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo show "${prev_rev}" + ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo show "${head_rev}" fatal "downgraded without --allow-downgrade?" fi assert_file_has_content err.txt "Upgrade.*is chronologically older" diff --git a/tests/libtest.sh b/tests/libtest.sh index 315c4df5..ca457fa2 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -488,6 +488,11 @@ EOF cd ${oldpwd} } +timestamp_of_commit() +{ + date --date="$(ostree --repo=$1 show $2 | grep -Ee '^Date: ' | sed -e 's,^Date: *,,')" '+%s' +} + os_repository_new_commit () { boot_checksum_iteration=${1:-0} @@ -529,6 +534,13 @@ os_repository_new_commit () echo "content iteration ${content_iteration}" > usr/bin/content-iteration ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" -b $branch -s "Build" + if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo rev-parse ${branch} 2>/dev/null; then + prevdate=$(timestamp_of_commit ${test_tmpdir}/testos-repo "${branch}"^) + newdate=$(timestamp_of_commit ${test_tmpdir}/testos-repo "${branch}") + if [ $((${prevdate} > ${newdate})) = 1 ]; then + fatal "clock skew detected writing commits: prev=${prevdate} new=${newdate}" + fi + fi cd ${test_tmpdir} } From 70ebb91aa3106e769a1f7cb285730388e8291c4a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 20 May 2020 10:45:45 -0400 Subject: [PATCH 137/177] tests/admin-test: Ensure that commits are 1s apart Otherwise the new check we added there to verify that upgrading without `--allow-downgrade` fails itself fails. See: https://github.com/ostreedev/ostree/pull/2099#issuecomment-629805840 --- tests/admin-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index c716d0a5..55da40c4 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -300,6 +300,8 @@ echo "ok no duplicate version strings in title" # Test upgrade with and without --override-commit # See https://github.com/GNOME/ostree/pull/147 +sleep 1 +os_repository_new_commit ${CMD_PREFIX} ostree pull --repo=sysroot/ostree/repo --commit-metadata-only --depth=-1 testos:testos/buildmaster/x86_64-runtime head_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime^^^^) From 191ce95ca1304d33fb712ab2eff079cf2e8b61ee Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 20 May 2020 15:57:10 +0000 Subject: [PATCH 138/177] admin-test: Show err.txt on unexpected failure In a CI run I think one of these `ostree show` commands is failing. While that output would be useful, the actual `err.txt` usually has what we want too. --- tests/admin-test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index c716d0a5..54b93321 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -306,8 +306,7 @@ prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buil assert_not_streq ${head_rev} ${prev_rev} # check that we can't "upgrade" to an older commit without --allow-downgrade if ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} 2> err.txt; then - ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo show "${prev_rev}" - ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo show "${head_rev}" + cat err.txt fatal "downgraded without --allow-downgrade?" fi assert_file_has_content err.txt "Upgrade.*is chronologically older" From e35b82fb891daee823fcce421ae8f1442b630ea2 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 20 May 2020 16:18:45 -0400 Subject: [PATCH 139/177] switchroot/remount: Neuter sysroot.readonly for now We're hitting issues with the read-only remounts racing with various services coming up. Let's neuter it for now until we rework how it works. See: https://github.com/coreos/fedora-coreos-tracker/issues/488 --- src/switchroot/ostree-remount.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 326b104f..00e21296 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -93,7 +93,10 @@ sysroot_is_configured_ro (void) if (!g_key_file_load_from_file (keyfile, config_path, 0, NULL)) return false; - return g_key_file_get_boolean (keyfile, "sysroot", "readonly", NULL); + if (g_key_file_get_boolean (keyfile, "sysroot", "readonly", NULL)) + puts ("Ignoring sysroot.readonly config; see https://github.com/coreos/fedora-coreos-tracker/issues/488."); + + return false; } int From 0487b498ad38534f98b5b0b09ac2442b3e0879b1 Mon Sep 17 00:00:00 2001 From: AJ Jordan Date: Thu, 21 May 2020 23:22:44 -0700 Subject: [PATCH 140/177] Fix typo --- docs/manual/repo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index 07cc1a93..8746163e 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -109,7 +109,7 @@ to override this, or set the `$OSTREE_REPO` environment variable. ## Refs Like git, OSTree uses the terminology "references" (abbreviated -"refs") which are text files that name (refer to) to particular +"refs") which are text files that name (refer to) particular commits. See the [Git Documentation](https://git-scm.com/book/en/v2/Git-Internals-Git-References) for information on how git uses them. Unlike git though, it doesn't From 6730acc3504fdc663b8d01a72427d8be695a3adb Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 20 May 2020 12:37:44 -0400 Subject: [PATCH 141/177] tests/admin-test: Fix --allow-downgrade check We were doing a check to verify that `ostree admin upgrade` wouldn't accept a downgrade without `--allow-downgrade`. However, there's no guarantee that the commit it's upgrading from is older than HEAD^ (what we're upgrading to). Specifically, if the test runs fast enough, the timestamps could be equal, since the lowest resolution is seconds. Rework the test so that we first upgrade to HEAD, which we're sure is at least 1 second apart from HEAD^, and *then* check that downgrade protection is enforced. We also can't use `rev-parse testos/buildmaster/x86_64-runtime` as a way to know what commit the host is sitting on since the ref might've gone ahead. Instead, just use `ostree admin status | head -n1`. (I played with using the `ostree/I/J/K` refs, but those depend on what the boot/subbootversion is and can easily change if we change previous tests). --- tests/admin-test.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 18ca0d95..3aab74cc 100644 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -302,23 +302,31 @@ echo "ok no duplicate version strings in title" # See https://github.com/GNOME/ostree/pull/147 sleep 1 os_repository_new_commit -${CMD_PREFIX} ostree pull --repo=sysroot/ostree/repo --commit-metadata-only --depth=-1 testos:testos/buildmaster/x86_64-runtime +# upgrade to the latest +${CMD_PREFIX} ostree admin upgrade --os=testos head_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) -prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime^^^^) +prev_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime^) assert_not_streq ${head_rev} ${prev_rev} -# check that we can't "upgrade" to an older commit without --allow-downgrade +# Don't use `ostree admin status | head -n 1` directly here because `head` +# exiting early might cause SIGPIPE to ostree, which with `set -euo pipefail` +# will cause us to exit. See: https://github.com/ostreedev/ostree/pull/2110. +${CMD_PREFIX} ostree admin status > status-out.txt +head -n 1 < status-out.txt > status.txt +assert_file_has_content status.txt ".* testos ${head_rev}.*" +# now, check that we can't downgrade to an older commit without --allow-downgrade if ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} 2> err.txt; then cat err.txt fatal "downgraded without --allow-downgrade?" fi assert_file_has_content err.txt "Upgrade.*is chronologically older" ${CMD_PREFIX} ostree admin upgrade --os=testos --override-commit=${prev_rev} --allow-downgrade -curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) - -assert_streq ${curr_rev} ${prev_rev} +${CMD_PREFIX} ostree admin status > status-out.txt +head -n 1 < status-out.txt > status.txt +assert_file_has_content status.txt ".* testos ${prev_rev}.*" ${CMD_PREFIX} ostree admin upgrade --os=testos -curr_rev=$(${CMD_PREFIX} ostree rev-parse --repo=sysroot/ostree/repo testos/buildmaster/x86_64-runtime) -assert_streq ${curr_rev} ${head_rev} +${CMD_PREFIX} ostree admin status > status-out.txt +head -n 1 < status-out.txt > status.txt +assert_file_has_content status.txt ".* testos ${head_rev}.*" echo "ok upgrade with and without override-commit" From 5cb9d0df38e7c24e9db999c2d4e6a76fbf758fa5 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 May 2020 20:43:23 +0000 Subject: [PATCH 142/177] pull: Add support for sign-verify= The goal here is to move the code towards a model where the *client* can explicitly specify which signature types are acceptable. We retain support for `sign-verify=true` for backwards compatibility. But in that configuration, a missing public key is just "no signatures found". With `sign-verify=ed25519` and no key configured, we can explicitly say `No keys found for required signapi type ed25519` which is much, much clearer. Implementation side, rather than maintaining `gboolean sign_verify` *and* `GPtrArray sign_verifiers`, just have the array. If it's `NULL` that means not to verify. Note that currently, an explicit list is an OR of signatures, not AND. In practice...I think most people are going to be using a single entry anyways. --- src/libostree/ostree-repo-pull-private.h | 16 +-- src/libostree/ostree-repo-pull-verify.c | 138 +++++++++++++++++++---- src/libostree/ostree-repo-pull.c | 76 ++++--------- tests/test-signed-pull.sh | 27 ++++- 4 files changed, 174 insertions(+), 83 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index 86d1ffee..fd17baee 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -67,8 +67,6 @@ typedef struct { gboolean gpg_verify; gboolean gpg_verify_summary; - gboolean sign_verify; - gboolean sign_verify_summary; gboolean require_static_deltas; gboolean disable_static_deltas; gboolean has_tombstone_commits; @@ -124,7 +122,8 @@ typedef struct { gboolean is_commit_only; OstreeRepoImportFlags importflags; - GPtrArray *signapi_verifiers; + GPtrArray *signapi_commit_verifiers; + GPtrArray *signapi_summary_verifiers; GPtrArray *dirs; @@ -140,11 +139,12 @@ typedef struct { GSource *idle_src; } OtPullData; -GPtrArray * -_signapi_verifiers_for_remote (OstreeRepo *repo, - const char *remote_name, - GError **error); - +gboolean +_signapi_init_for_remote (OstreeRepo *repo, + const char *remote_name, + GPtrArray **out_commit_verifiers, + GPtrArray **out_summary_verifiers, + GError **error); gboolean _sign_verify_for_remote (GPtrArray *signers, GBytes *signed_data, diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c index 36d877ac..ab680daf 100644 --- a/src/libostree/ostree-repo-pull-verify.c +++ b/src/libostree/ostree-repo-pull-verify.c @@ -71,6 +71,7 @@ static gboolean _signapi_load_public_keys (OstreeSign *sign, OstreeRepo *repo, const gchar *remote_name, + gboolean required, GError **error) { g_autofree gchar *pk_ascii = NULL; @@ -95,6 +96,8 @@ _signapi_load_public_keys (OstreeSign *sign, * and call it here for loading with method and file structure * specific for signature type. */ + if (required) + return glnx_throw (error, "No keys found for required signapi type %s", ostree_sign_get_name (sign)); return TRUE; } @@ -142,31 +145,120 @@ _signapi_load_public_keys (OstreeSign *sign, return TRUE; } -/* Create a new array of OstreeSign objects and load the public - * keys as described by the remote configuration. - */ -GPtrArray * -_signapi_verifiers_for_remote (OstreeRepo *repo, - const char *remote_name, - GError **error) +static gboolean +string_is_gkeyfile_truthy (const char *value, + gboolean *out_truth) { - g_autoptr(GPtrArray) signers = ostree_sign_get_all (); - g_assert_cmpuint (signers->len, >=, 1); - for (guint i = 0; i < signers->len; i++) + /* See https://gitlab.gnome.org/GNOME/glib/-/blob/20fb5bf868added5aec53c013ae85ec78ba2eedc/glib/gkeyfile.c#L4528 */ + if (g_str_equal (value, "true") || g_str_equal (value, "1")) { - OstreeSign *sign = signers->pdata[i]; - /* Try to load public key(s) according remote's configuration */ - if (!_signapi_load_public_keys (sign, repo, remote_name, error)) - return FALSE; + *out_truth = TRUE; + return TRUE; } - return g_steal_pointer (&signers); + else if (g_str_equal (value, "false") || g_str_equal (value, "0")) + { + *out_truth = FALSE; + return TRUE; + } + return FALSE; } -/* Iterate over the configured signers, and require the commit is signed +static gboolean +verifiers_from_config (OstreeRepo *repo, + const char *remote_name, + const char *key, + GPtrArray **out_verifiers, + GError **error) +{ + g_autoptr(GPtrArray) verifiers = NULL; + + g_autofree char *raw_value = NULL; + if (!ostree_repo_get_remote_option (repo, remote_name, + key, NULL, + &raw_value, error)) + return FALSE; + if (raw_value == NULL || g_str_equal (raw_value, "")) + { + *out_verifiers = NULL; + return TRUE; + } + gboolean sign_verify_bool = FALSE; + /* Is the value "truthy" according to GKeyFile's rules? If so, + * then we take this to be "accept signatures from any compiled + * type that happens to have keys configured". + */ + if (string_is_gkeyfile_truthy (raw_value, &sign_verify_bool)) + { + if (sign_verify_bool) + { + verifiers = ostree_sign_get_all (); + for (guint i = 0; i < verifiers->len; i++) + { + OstreeSign *sign = verifiers->pdata[i]; + /* Try to load public key(s) according remote's configuration; + * this one is optional. + */ + if (!_signapi_load_public_keys (sign, repo, remote_name, FALSE, error)) + return FALSE; + } + } + } + else + { + /* If the value isn't "truthy", then it must be an explicit list */ + g_auto(GStrv) sign_types = NULL; + if (!ostree_repo_get_remote_list_option (repo, remote_name, + key, &sign_types, + error)) + return FALSE; + verifiers = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); + for (char **iter = sign_types; iter && *iter; iter++) + { + const char *sign_type = *iter; + OstreeSign *verifier = ostree_sign_get_by_name (sign_type, error); + if (!verifier) + return FALSE; + if (!_signapi_load_public_keys (verifier, repo, remote_name, TRUE, error)) + return FALSE; + g_ptr_array_add (verifiers, verifier); + } + g_assert_cmpuint (verifiers->len, >=, 1); + } + + *out_verifiers = g_steal_pointer (&verifiers); + return TRUE; +} + +/* Create a new array of OstreeSign objects and load the public + * keys as described by the remote configuration. If the + * remote does not have signing verification enabled, then + * the resulting verifier list will be NULL. + */ +gboolean +_signapi_init_for_remote (OstreeRepo *repo, + const char *remote_name, + GPtrArray **out_commit_verifiers, + GPtrArray **out_summary_verifiers, + GError **error) +{ + g_autoptr(GPtrArray) commit_verifiers = NULL; + g_autoptr(GPtrArray) summary_verifiers = NULL; + + if (!verifiers_from_config (repo, remote_name, "sign-verify", &commit_verifiers, error)) + return FALSE; + if (!verifiers_from_config (repo, remote_name, "sign-verify-summary", &summary_verifiers, error)) + return FALSE; + + ot_transfer_out_value (out_commit_verifiers, &commit_verifiers); + ot_transfer_out_value (out_summary_verifiers, &summary_verifiers); + return TRUE; +} + +/* Iterate over the configured verifiers, and require the commit is signed * by at least one. */ gboolean -_sign_verify_for_remote (GPtrArray *signers, +_sign_verify_for_remote (GPtrArray *verifiers, GBytes *signed_data, GVariant *metadata, GError **error) @@ -175,10 +267,10 @@ _sign_verify_for_remote (GPtrArray *signers, g_autoptr (GError) last_sig_error = NULL; gboolean found_sig = FALSE; - g_assert_cmpuint (signers->len, >=, 1); - for (guint i = 0; i < signers->len; i++) + g_assert_cmpuint (verifiers->len, >=, 1); + for (guint i = 0; i < verifiers->len; i++) { - OstreeSign *sign = signers->pdata[i]; + OstreeSign *sign = verifiers->pdata[i]; const gchar *signature_key = ostree_sign_metadata_key (sign); GVariantType *signature_format = (GVariantType *) ostree_sign_metadata_format (sign); g_autoptr (GVariant) signatures = @@ -257,7 +349,7 @@ _verify_unwritten_commit (OtPullData *pull_data, GError **error) { - if (pull_data->gpg_verify || pull_data->sign_verify) + if (pull_data->gpg_verify || pull_data->signapi_commit_verifiers) /* Shouldn't happen, but see comment in process_gpg_verify_result() */ if (g_hash_table_contains (pull_data->verified_commits, checksum)) return TRUE; @@ -284,13 +376,13 @@ _verify_unwritten_commit (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ - if (pull_data->sign_verify) + if (pull_data->signapi_commit_verifiers) { /* Nothing to check if detached metadata is absent */ if (detached_metadata == NULL) return glnx_throw (error, "Can't verify commit without detached metadata"); - if (!_sign_verify_for_remote (pull_data->signapi_verifiers, signed_data, detached_metadata, error)) + if (!_sign_verify_for_remote (pull_data->signapi_commit_verifiers, signed_data, detached_metadata, error)) return glnx_prefix_error (error, "Can't verify commit"); /* Mark the commit as verified to avoid double verification diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 7116c3dc..5a57bfa6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1537,17 +1537,16 @@ scan_commit_object (OtPullData *pull_data, } #endif /* OSTREE_DISABLE_GPGME */ - if (pull_data->sign_verify && + if (pull_data->signapi_commit_verifiers && !g_hash_table_contains (pull_data->verified_commits, checksum)) { g_autoptr(GError) last_verification_error = NULL; gboolean found_any_signature = FALSE; gboolean found_valid_signature = FALSE; - g_assert (pull_data->signapi_verifiers); - for (guint i = 0; i < pull_data->signapi_verifiers->len; i++) + for (guint i = 0; i < pull_data->signapi_commit_verifiers->len; i++) { - OstreeSign *sign = pull_data->signapi_verifiers->pdata[i]; + OstreeSign *sign = pull_data->signapi_commit_verifiers->pdata[i]; found_any_signature = TRUE; @@ -3552,31 +3551,6 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!ostree_repo_remote_get_gpg_verify_summary (self, pull_data->remote_name, &pull_data->gpg_verify_summary, error)) goto out; - /* signapi differs from GPG in that it can only be explicitly *disabled* - * transiently during pulls, not enabled. - */ - if (disable_sign_verify) - { - pull_data->sign_verify = FALSE; - } - else - { - if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, - "sign-verify", FALSE, - &pull_data->sign_verify, error)) - goto out; - } - if (disable_sign_verify_summary) - { - pull_data->sign_verify_summary = FALSE; - } - else - { - if (!ostree_repo_get_remote_boolean_option (self, pull_data->remote_name, - "sign-verify-summary", FALSE, - &pull_data->sign_verify_summary, error)) - goto out; - } /* NOTE: If changing this, see the matching implementation in * ostree-sysroot-upgrader.c @@ -3595,13 +3569,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, } } - if (pull_data->sign_verify || pull_data->sign_verify_summary) + if (pull_data->remote_name && !(disable_sign_verify && disable_sign_verify_summary)) { - g_assert (pull_data->remote_name != NULL); - pull_data->signapi_verifiers = _signapi_verifiers_for_remote (pull_data->repo, pull_data->remote_name, error); - if (!pull_data->signapi_verifiers) - goto out; - g_assert_cmpint (pull_data->signapi_verifiers->len, >=, 1); + if (!_signapi_init_for_remote (pull_data->repo, pull_data->remote_name, + &pull_data->signapi_commit_verifiers, + &pull_data->signapi_summary_verifiers, + error)) + return FALSE; } pull_data->phase = OSTREE_PULL_PHASE_FETCHING_REFS; @@ -3967,9 +3941,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, } #endif /* OSTREE_DISABLE_GPGME */ - if (pull_data->sign_verify_summary) + if (pull_data->signapi_summary_verifiers) { - if (!bytes_sig && pull_data->sign_verify_summary) + if (!bytes_sig && pull_data->signapi_summary_verifiers) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); @@ -3984,8 +3958,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, bytes_sig, FALSE); - g_assert (pull_data->signapi_verifiers); - if (!_sign_verify_for_remote (pull_data->signapi_verifiers, bytes_summary, signatures, &temp_error)) + g_assert (pull_data->signapi_summary_verifiers); + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, &temp_error)) { if (summary_from_cache) { @@ -4014,7 +3988,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (!_sign_verify_for_remote (pull_data->signapi_verifiers, bytes_summary, signatures, error)) + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, error)) goto out; } else @@ -4536,7 +4510,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_string_append_printf (msg, "\nsecurity: GPG: %s ", gpg_verify_state); const char *sign_verify_state; - sign_verify_state = (pull_data->sign_verify ? "commit" : "disabled"); + sign_verify_state = (pull_data->signapi_commit_verifiers ? "commit" : "disabled"); g_string_append_printf (msg, "\nsecurity: SIGN: %s ", sign_verify_state); OstreeFetcherURI *first_uri = pull_data->meta_mirrorlist->pdata[0]; @@ -4629,7 +4603,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_free (pull_data->remote_refspec_name); g_free (pull_data->remote_name); g_free (pull_data->append_user_agent); - g_clear_pointer (&pull_data->signapi_verifiers, (GDestroyNotify) g_ptr_array_unref); + g_clear_pointer (&pull_data->signapi_commit_verifiers, (GDestroyNotify) g_ptr_array_unref); + g_clear_pointer (&pull_data->signapi_summary_verifiers, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->content_mirrorlist, (GDestroyNotify) g_ptr_array_unref); g_clear_pointer (&pull_data->summary_data, (GDestroyNotify) g_bytes_unref); @@ -6050,7 +6025,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, g_autoptr(GBytes) summary = NULL; g_autoptr(GBytes) signatures = NULL; gboolean gpg_verify_summary; - gboolean sign_verify_summary; + g_autoptr(GPtrArray) signapi_summary_verifiers = NULL; gboolean ret = FALSE; gboolean summary_is_from_cache; @@ -6107,11 +6082,12 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, } } - if (!ostree_repo_get_remote_boolean_option (self, name, "sign-verify-summary", - FALSE, &sign_verify_summary, error)) - goto out; + if (!_signapi_init_for_remote (self, name, NULL, + &signapi_summary_verifiers, + error)) + goto out; - if (sign_verify_summary) + if (signapi_summary_verifiers) { if (summary == NULL) { @@ -6134,10 +6110,8 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, signatures, FALSE); - g_autoptr(GPtrArray) signapi_verifiers = _signapi_verifiers_for_remote (self, name, error); - if (!signapi_verifiers) - goto out; - if (!_sign_verify_for_remote (signapi_verifiers, summary, sig_variant, error)) + + if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, error)) goto out; } } diff --git a/tests/test-signed-pull.sh b/tests/test-signed-pull.sh index b207eac2..fe78321a 100755 --- a/tests/test-signed-pull.sh +++ b/tests/test-signed-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..16" +echo "1..20" # This is explicitly opt in for testing export OSTREE_DUMMY_SIGN_ENABLED=1 @@ -102,6 +102,31 @@ test_signed_pull "dummy" "" repo_init --sign-verify=dummy=inline:${DUMMYSIGN} test_signed_pull "dummy" "from remote opt" +# And now explicitly limit it to dummy +repo_init +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.sign-verify dummy +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy-key "${DUMMYSIGN}" +test_signed_pull "dummy" "explicit value" + +# dummy, but no key configured +repo_init +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.sign-verify dummy +if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.txt; then + assert_not_reached "pull with nosuchsystem succeeded" +fi +assert_file_has_content err.txt 'No keys found for required signapi type dummy' +echo "ok explicit dummy but unconfigured" + +# Set it to an unknown explicit value +repo_init +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.sign-verify nosuchsystem; +${CMD_PREFIX} ostree --repo=repo config set 'remote "origin"'.verification-dummy-key "${DUMMYSIGN}" +if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.txt; then + assert_not_reached "pull with nosuchsystem succeeded" +fi +assert_file_has_content err.txt 'Requested signature type is not implemented' +echo "ok pull failure for unknown system" + repo_init if ${CMD_PREFIX} ostree --repo=repo remote add other --sign-verify=trustme=inline:ok http://localhost 2>err.txt; then assert_not_reached "remote add with invalid keytype succeeded" From 35642259175973617da937f3cab6ce5f13c95077 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 24 May 2020 15:25:08 +0000 Subject: [PATCH 143/177] Move ro /sysroot bind mount of /etc into initramfs We recently disabled the read-only /sysroot handling: https://github.com/ostreedev/ostree/pull/2108/commits/e35b82fb891daee823fcce421ae8f1442b630ea2 The core problem was that a lot of services run early in the real root and want write access to things like `/var` and `/etc`. In trying to do remounts while the system is running we introduce too many race conditions. Instead, just make the `/etc` bind mount in the initramfs right after we set up the main root. This is much more natural really, and avoids all race conditions since nothing is running in the sysroot yet. The main awkward part is that since we're not linking `ostree-prepare-root` to GLib (yet) we have a hacky parser for the config file. But, this is going to be fine I think. In order to avoid parsing the config twice, pass state from `ostree-prepare-root` to `ostree-remount` via a file in `/run`. --- src/switchroot/ostree-mount-util.h | 4 +- src/switchroot/ostree-prepare-root.c | 69 ++++++++++++++++++++++++++++ src/switchroot/ostree-remount.c | 39 ++-------------- 3 files changed, 75 insertions(+), 37 deletions(-) diff --git a/src/switchroot/ostree-mount-util.h b/src/switchroot/ostree-mount-util.h index 0b40bb40..fb2d02b4 100644 --- a/src/switchroot/ostree-mount-util.h +++ b/src/switchroot/ostree-mount-util.h @@ -30,11 +30,13 @@ #include #include #include +#include #define INITRAMFS_MOUNT_VAR "/run/ostree/initramfs-mount-var" +#define _OSTREE_SYSROOT_READONLY_STAMP "/run/ostree-sysroot-ro.stamp" static inline int -path_is_on_readonly_fs (char *path) +path_is_on_readonly_fs (const char *path) { struct statvfs stvfsbuf; diff --git a/src/switchroot/ostree-prepare-root.c b/src/switchroot/ostree-prepare-root.c index c25d3fe9..8a68e1f4 100644 --- a/src/switchroot/ostree-prepare-root.c +++ b/src/switchroot/ostree-prepare-root.c @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,47 @@ /* Initialized early in main */ static bool running_as_pid1; +static inline bool +sysroot_is_configured_ro (const char *sysroot) +{ + char * config_path = NULL; + assert (asprintf (&config_path, "%s/ostree/repo/config", sysroot) != -1); + FILE *f = fopen(config_path, "r"); + if (!f) + { + fprintf (stderr, "Missing expected repo config: %s\n", config_path); + free (config_path); + return false; + } + free (config_path); + + bool ret = false; + char *line = NULL; + size_t len = 0; + ssize_t nread; + /* Note getline() will reuse the previous buffer */ + bool in_sysroot = false; + while ((nread = getline (&line, &len, f)) != -1) + { + /* This is an awful hack to avoid depending on GLib in the + * initramfs right now. + */ + if (strstr (line, "[sysroot]") == line) + in_sysroot = true; + else if (*line == '[') + in_sysroot = false; + else if (in_sysroot && strstr (line, "readonly=true") == line) + { + ret = true; + break; + } + } + + fclose (f); + free (line); + return ret; +} + static char* resolve_deploy_path (const char * root_mountpoint) { @@ -192,6 +234,33 @@ main(int argc, char *argv[]) if (chdir (deploy_path) < 0) err (EXIT_FAILURE, "failed to chdir to deploy_path"); + /* Query the repository configuration - this is an operating system builder + * choice. More info: https://github.com/ostreedev/ostree/pull/1767 + */ + const bool sysroot_readonly = sysroot_is_configured_ro (root_arg); + const bool sysroot_currently_writable = !path_is_on_readonly_fs (root_arg); + +#ifdef USE_LIBSYSTEMD + sd_journal_send ("MESSAGE=sysroot configured read-only: %d, currently writable: %d", + (int)sysroot_readonly, (int)sysroot_currently_writable, NULL); +#endif + if (sysroot_readonly) + { + if (!sysroot_currently_writable) + errx (EXIT_FAILURE, "sysroot=readonly currently requires writable / in initramfs"); + /* Now, /etc is not normally a bind mount, but if we have a readonly + * sysroot, we still need a writable /etc. And to avoid race conditions + * we ensure it's writable in the initramfs, before we switchroot at all. + */ + if (mount ("/etc", "/etc", NULL, MS_BIND, NULL) < 0) + err (EXIT_FAILURE, "failed to make /etc a bind mount"); + /* Pass on the fact that we discovered a readonly sysroot to ostree-remount.service */ + int fd = open (_OSTREE_SYSROOT_READONLY_STAMP, O_WRONLY | O_CREAT | O_CLOEXEC, 0644); + if (fd < 0) + err (EXIT_FAILURE, "failed to create %s", _OSTREE_SYSROOT_READONLY_STAMP); + (void) close (fd); + } + /* Default to true, but in the systemd case, default to false because it's handled by * ostree-system-generator. */ bool mount_var = true; diff --git a/src/switchroot/ostree-remount.c b/src/switchroot/ostree-remount.c index 00e21296..5c313c87 100644 --- a/src/switchroot/ostree-remount.c +++ b/src/switchroot/ostree-remount.c @@ -81,24 +81,6 @@ do_remount (const char *target, printf ("Remounted %s: %s\n", writable ? "rw" : "ro", target); } -static bool -sysroot_is_configured_ro (void) -{ - struct stat stbuf; - static const char config_path[] = "/ostree/repo/config"; - if (stat (config_path, &stbuf) != 0) - return false; - - g_autoptr(GKeyFile) keyfile = g_key_file_new (); - if (!g_key_file_load_from_file (keyfile, config_path, 0, NULL)) - return false; - - if (g_key_file_get_boolean (keyfile, "sysroot", "readonly", NULL)) - puts ("Ignoring sysroot.readonly config; see https://github.com/coreos/fedora-coreos-tracker/issues/488."); - - return false; -} - int main(int argc, char *argv[]) { @@ -124,25 +106,10 @@ main(int argc, char *argv[]) exit (EXIT_SUCCESS); } - /* Query the repository configuration - this is an operating system builder - * choice. - * */ - const bool sysroot_readonly = sysroot_is_configured_ro (); - - /* Mount the sysroot read-only if we're configured to do so. - * Note we only get here if / is already writable. - */ - do_remount ("/sysroot", !sysroot_readonly); - - if (sysroot_readonly) + /* Handle remounting /sysroot read-only now */ + if (unlink (_OSTREE_SYSROOT_READONLY_STAMP) == 0) { - /* Now, /etc is not normally a bind mount, but remounting the - * sysroot above made it read-only since it's on the same filesystem. - * Make it a self-bind mount, so we can then mount it read-write. - */ - if (mount ("/etc", "/etc", NULL, MS_BIND, NULL) < 0) - err (EXIT_FAILURE, "failed to make /etc a bind mount"); - do_remount ("/etc", true); + do_remount ("/sysroot", false); } /* If /var was created as as an OSTree default bind mount (instead of being a separate filesystem) From 718cca8055e437a4872ab12621ef78d87535bda0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 27 May 2020 12:52:07 +0000 Subject: [PATCH 144/177] tests/kola: Move to tests/kolainst Follow the precedent set in https://github.com/coreos/rpm-ostree/pull/2106 and rename the directory, to more clearly move away from the "uninstalled" test model. Prep for Rust-based tests. --- .cci.jenkinsfile | 5 ++--- tests/{kola => kolainst}/Makefile | 0 tests/{kola => kolainst}/README.md | 0 tests/{kola => kolainst}/destructive/data | 0 tests/{kola => kolainst}/destructive/itest-bare-root.sh | 0 tests/{kola => kolainst}/destructive/itest-deploy-selinux.sh | 0 tests/{kola => kolainst}/destructive/itest-label-selinux.sh | 0 tests/{kola => kolainst}/destructive/staged-delay.sh | 0 tests/{kola => kolainst}/destructive/staged-deploy.sh | 0 tests/{kola => kolainst}/destructive/var-mount.sh | 0 tests/{kola => kolainst}/libinsttest.sh | 0 tests/{kola => kolainst}/libtest-core.sh | 0 tests/{kola => kolainst}/nondestructive/data | 0 tests/{kola => kolainst}/nondestructive/itest-bare-unit.sh | 0 .../nondestructive/itest-bare-user-root.sh | 0 .../nondestructive/itest-bareuser-nouserxattrs.sh | 0 .../{kola => kolainst}/nondestructive/itest-payload-link.sh | 0 tests/{kola => kolainst}/nondestructive/itest-pull-space.sh | 0 tests/{kola => kolainst}/nondestructive/itest-pull.sh | 0 tests/{kola => kolainst}/nondestructive/itest-remotes.sh | 0 tests/{kola => kolainst}/nondestructive/libtest-core.sh | 0 tests/libtest-core.sh | 2 +- 22 files changed, 3 insertions(+), 4 deletions(-) rename tests/{kola => kolainst}/Makefile (100%) rename tests/{kola => kolainst}/README.md (100%) rename tests/{kola => kolainst}/destructive/data (100%) rename tests/{kola => kolainst}/destructive/itest-bare-root.sh (100%) rename tests/{kola => kolainst}/destructive/itest-deploy-selinux.sh (100%) rename tests/{kola => kolainst}/destructive/itest-label-selinux.sh (100%) rename tests/{kola => kolainst}/destructive/staged-delay.sh (100%) rename tests/{kola => kolainst}/destructive/staged-deploy.sh (100%) rename tests/{kola => kolainst}/destructive/var-mount.sh (100%) rename tests/{kola => kolainst}/libinsttest.sh (100%) rename tests/{kola => kolainst}/libtest-core.sh (100%) rename tests/{kola => kolainst}/nondestructive/data (100%) rename tests/{kola => kolainst}/nondestructive/itest-bare-unit.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-bare-user-root.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-bareuser-nouserxattrs.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-payload-link.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-pull-space.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-pull.sh (100%) rename tests/{kola => kolainst}/nondestructive/itest-remotes.sh (100%) rename tests/{kola => kolainst}/nondestructive/libtest-core.sh (100%) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 68656cbd..d459a8db 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -76,9 +76,8 @@ parallel fcos: { rmdir insttree coreos-assembler fetch coreos-assembler build - # Install the tests, then be sure they're not run again - make -C tests/kola install - rm tests/kola -rf + # Install the tests + make -C tests/kolainst install """) } fcosKola(cosaDir: "${env.WORKSPACE}") diff --git a/tests/kola/Makefile b/tests/kolainst/Makefile similarity index 100% rename from tests/kola/Makefile rename to tests/kolainst/Makefile diff --git a/tests/kola/README.md b/tests/kolainst/README.md similarity index 100% rename from tests/kola/README.md rename to tests/kolainst/README.md diff --git a/tests/kola/destructive/data b/tests/kolainst/destructive/data similarity index 100% rename from tests/kola/destructive/data rename to tests/kolainst/destructive/data diff --git a/tests/kola/destructive/itest-bare-root.sh b/tests/kolainst/destructive/itest-bare-root.sh similarity index 100% rename from tests/kola/destructive/itest-bare-root.sh rename to tests/kolainst/destructive/itest-bare-root.sh diff --git a/tests/kola/destructive/itest-deploy-selinux.sh b/tests/kolainst/destructive/itest-deploy-selinux.sh similarity index 100% rename from tests/kola/destructive/itest-deploy-selinux.sh rename to tests/kolainst/destructive/itest-deploy-selinux.sh diff --git a/tests/kola/destructive/itest-label-selinux.sh b/tests/kolainst/destructive/itest-label-selinux.sh similarity index 100% rename from tests/kola/destructive/itest-label-selinux.sh rename to tests/kolainst/destructive/itest-label-selinux.sh diff --git a/tests/kola/destructive/staged-delay.sh b/tests/kolainst/destructive/staged-delay.sh similarity index 100% rename from tests/kola/destructive/staged-delay.sh rename to tests/kolainst/destructive/staged-delay.sh diff --git a/tests/kola/destructive/staged-deploy.sh b/tests/kolainst/destructive/staged-deploy.sh similarity index 100% rename from tests/kola/destructive/staged-deploy.sh rename to tests/kolainst/destructive/staged-deploy.sh diff --git a/tests/kola/destructive/var-mount.sh b/tests/kolainst/destructive/var-mount.sh similarity index 100% rename from tests/kola/destructive/var-mount.sh rename to tests/kolainst/destructive/var-mount.sh diff --git a/tests/kola/libinsttest.sh b/tests/kolainst/libinsttest.sh similarity index 100% rename from tests/kola/libinsttest.sh rename to tests/kolainst/libinsttest.sh diff --git a/tests/kola/libtest-core.sh b/tests/kolainst/libtest-core.sh similarity index 100% rename from tests/kola/libtest-core.sh rename to tests/kolainst/libtest-core.sh diff --git a/tests/kola/nondestructive/data b/tests/kolainst/nondestructive/data similarity index 100% rename from tests/kola/nondestructive/data rename to tests/kolainst/nondestructive/data diff --git a/tests/kola/nondestructive/itest-bare-unit.sh b/tests/kolainst/nondestructive/itest-bare-unit.sh similarity index 100% rename from tests/kola/nondestructive/itest-bare-unit.sh rename to tests/kolainst/nondestructive/itest-bare-unit.sh diff --git a/tests/kola/nondestructive/itest-bare-user-root.sh b/tests/kolainst/nondestructive/itest-bare-user-root.sh similarity index 100% rename from tests/kola/nondestructive/itest-bare-user-root.sh rename to tests/kolainst/nondestructive/itest-bare-user-root.sh diff --git a/tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh b/tests/kolainst/nondestructive/itest-bareuser-nouserxattrs.sh similarity index 100% rename from tests/kola/nondestructive/itest-bareuser-nouserxattrs.sh rename to tests/kolainst/nondestructive/itest-bareuser-nouserxattrs.sh diff --git a/tests/kola/nondestructive/itest-payload-link.sh b/tests/kolainst/nondestructive/itest-payload-link.sh similarity index 100% rename from tests/kola/nondestructive/itest-payload-link.sh rename to tests/kolainst/nondestructive/itest-payload-link.sh diff --git a/tests/kola/nondestructive/itest-pull-space.sh b/tests/kolainst/nondestructive/itest-pull-space.sh similarity index 100% rename from tests/kola/nondestructive/itest-pull-space.sh rename to tests/kolainst/nondestructive/itest-pull-space.sh diff --git a/tests/kola/nondestructive/itest-pull.sh b/tests/kolainst/nondestructive/itest-pull.sh similarity index 100% rename from tests/kola/nondestructive/itest-pull.sh rename to tests/kolainst/nondestructive/itest-pull.sh diff --git a/tests/kola/nondestructive/itest-remotes.sh b/tests/kolainst/nondestructive/itest-remotes.sh similarity index 100% rename from tests/kola/nondestructive/itest-remotes.sh rename to tests/kolainst/nondestructive/itest-remotes.sh diff --git a/tests/kola/nondestructive/libtest-core.sh b/tests/kolainst/nondestructive/libtest-core.sh similarity index 100% rename from tests/kola/nondestructive/libtest-core.sh rename to tests/kolainst/nondestructive/libtest-core.sh diff --git a/tests/libtest-core.sh b/tests/libtest-core.sh index 11131fca..3c181880 120000 --- a/tests/libtest-core.sh +++ b/tests/libtest-core.sh @@ -1 +1 @@ -kola/libtest-core.sh \ No newline at end of file +kolainst/libtest-core.sh \ No newline at end of file From 1f637bf34103746ab07f359d5488224134a16a08 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 28 Mar 2020 14:04:31 +0000 Subject: [PATCH 145/177] Add new Rust-based tests There's a lot going on here. First, this is intended to run nicely as part of the new [cosa/kola ext-tests](https://github.com/coreos/coreos-assembler/pull/1252). With Rust we can get one big static binary that we can upload, and include a webserver as part of the binary. This way we don't need to do the hack of running a container with Python or whatever. Now, what's even better about Rust for this is that it has macros, and specifically we are using [commandspec](https://github.com/tcr/commandspec/) which allows us to "inline" shell script. I think the macros could be even better, but this shows how we can intermix pure Rust code along with using shell safely enough. We're using my fork of commandspec because the upstream hasn't merged [a few PRs](https://github.com/tcr/commandspec/pulls?q=is%3Apr+author%3Acgwalters+). This model is intended to replace *both* some of our `make check` tests as well. Oh, and this takes the obvious step of using the Rust OSTree bindings as part of our tests. Currently the "commandspec tests" and "API tests" are separate, but nothing stops us from intermixing them if we wanted. I haven't yet tried to write destructive tests with this but I think it will go well. --- .cci.jenkinsfile | 2 +- tests/basic-test.sh | 32 +--- tests/inst/.gitignore | 2 + tests/inst/Cargo.toml | 42 +++++ tests/inst/itest-macro/Cargo.toml | 14 ++ tests/inst/itest-macro/src/itest-macro.rs | 32 ++++ tests/inst/src/insttest.rs | 46 ++++++ tests/inst/src/repobin.rs | 121 +++++++++++++++ tests/inst/src/sysroot.rs | 33 ++++ tests/inst/src/test.rs | 180 ++++++++++++++++++++++ tests/kola/nondestructive/.gitignore | 2 + tests/kolainst/Makefile | 4 +- tests/runkola | 18 +++ 13 files changed, 495 insertions(+), 33 deletions(-) create mode 100644 tests/inst/.gitignore create mode 100644 tests/inst/Cargo.toml create mode 100644 tests/inst/itest-macro/Cargo.toml create mode 100644 tests/inst/itest-macro/src/itest-macro.rs create mode 100644 tests/inst/src/insttest.rs create mode 100644 tests/inst/src/repobin.rs create mode 100644 tests/inst/src/sysroot.rs create mode 100644 tests/inst/src/test.rs create mode 100644 tests/kola/nondestructive/.gitignore create mode 100755 tests/runkola diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index d459a8db..4315a1d0 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -62,7 +62,7 @@ codestyle: { // Build FCOS and do a kola basic run stage("More builds and test") { parallel fcos: { - cosaPod(runAsUser: 0, memory: "2048Mi", cpu: "2") { + cosaPod(buildroot: true, runAsUser: 0, memory: "3072Mi", cpu: "4") { stage("Build FCOS") { checkout scm unstash 'build' diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 97cd05e2..fc193f4f 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((89 + ${extra_basic_tests:-0}))" +echo "1..$((86 + ${extra_basic_tests:-0}))" CHECKOUT_U_ARG="" CHECKOUT_H_ARGS="-H" @@ -1031,17 +1031,6 @@ stat '--format=%Y' test2-checkout/baz/deeper > deeper-mtime assert_file_has_content deeper-mtime 0 echo "ok content mtime" -cd ${test_tmpdir} -rm -rf test2-checkout -mkdir -p test2-checkout -cd test2-checkout -mkfifo afifo -if $OSTREE commit ${COMMIT_ARGS} -b test2 -s "Attempt to commit a FIFO" 2>../errmsg; then - assert_not_reached "Committing a FIFO unexpetedly succeeded!" - assert_file_has_content ../errmsg "Unsupported file type" -fi -echo "ok commit of fifo was rejected" - cd ${test_tmpdir} rm repo2 -rf mkdir repo2 @@ -1180,22 +1169,3 @@ if test "$(id -u)" != "0"; then else echo "ok # SKIP not run when root" fi - -cd ${test_tmpdir} -rm -rf test2-checkout -mkdir -p test2-checkout -cd test2-checkout -touch blah -stat --printf="%.Y\n" ${test_tmpdir}/repo > ${test_tmpdir}/timestamp-orig.txt -$OSTREE commit ${COMMIT_ARGS} -b test2 -s "Should bump the mtime" -stat --printf="%.Y\n" ${test_tmpdir}/repo > ${test_tmpdir}/timestamp-new.txt -cd .. -if cmp timestamp-{orig,new}.txt; then - assert_not_reached "failed to update mtime on repo" -fi -echo "ok mtime updated" - -cd ${test_tmpdir} -$OSTREE init --mode=bare --repo=repo-extensions -assert_has_dir repo-extensions/extensions -echo "ok extensions dir" diff --git a/tests/inst/.gitignore b/tests/inst/.gitignore new file mode 100644 index 00000000..2c96eb1b --- /dev/null +++ b/tests/inst/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock diff --git a/tests/inst/Cargo.toml b/tests/inst/Cargo.toml new file mode 100644 index 00000000..a3838922 --- /dev/null +++ b/tests/inst/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "ostree-test" +version = "0.1.0" +authors = ["Colin Walters "] +edition = "2018" + +[[bin]] +name = "ostree-test" +path = "src/insttest.rs" + +[dependencies] +clap = "2.32.0" +structopt = "0.2" +commandspec = "0.12.2" +anyhow = "1.0" +tempfile = "3.1.0" +gio = "0.8" +ostree = { version = "0.7.1", features = ["v2020_1"] } +libtest-mimic = "0.2.0" +twoway = "0.2.1" +hyper = "0.13" +futures = "0.3.4" +http = "0.2.0" +hyper-staticfile = "0.5.1" +tokio = { version = "0.2", features = ["full"] } +futures-util = "0.3.1" +base64 = "0.12.0" +procspawn = "0.8" +proc-macro2 = "0.4" +quote = "0.6" +syn = "0.15" +linkme = "0.2" + +itest-macro = { path = "itest-macro" } + +with-procspawn-tempdir = { git = "https://github.com/cgwalters/with-procspawn-tempdir" } +#with-procspawn-tempdir = { path = "/var/srv/walters/src/github/cgwalters/with-procspawn-tempdir" } + +# See https://github.com/tcr/commandspec/pulls?q=is%3Apr+author%3Acgwalters+ +[patch.crates-io] +commandspec = { git = "https://github.com/cgwalters/commandspec", branch = 'walters-master' } +#commandspec = { path = "/var/srv/walters/src/github/tcr/commandspec" } diff --git a/tests/inst/itest-macro/Cargo.toml b/tests/inst/itest-macro/Cargo.toml new file mode 100644 index 00000000..54494d29 --- /dev/null +++ b/tests/inst/itest-macro/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "itest-macro" +version = "0.1.0" +edition = "2018" + +[lib] +proc-macro = true +path = "src/itest-macro.rs" + +[dependencies] +quote = "1.0.3" +proc-macro2 = "1.0.10" +syn = { version = "1.0.3", features = ["full"] } +anyhow = "1.0" diff --git a/tests/inst/itest-macro/src/itest-macro.rs b/tests/inst/itest-macro/src/itest-macro.rs new file mode 100644 index 00000000..42b99581 --- /dev/null +++ b/tests/inst/itest-macro/src/itest-macro.rs @@ -0,0 +1,32 @@ +extern crate proc_macro; + +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; + +/// Wraps function using `procspawn` to allocate a new temporary directory, +/// make it the process' working directory, and run the function. +#[proc_macro_attribute] +pub fn itest(attrs: TokenStream, input: TokenStream) -> TokenStream { + let attrs = syn::parse_macro_input!(attrs as syn::AttributeArgs); + if attrs.len() > 0 { + return syn::Error::new_spanned(&attrs[0], "itest takes no attributes") + .to_compile_error() + .into(); + } + let func = syn::parse_macro_input!(input as syn::ItemFn); + let fident = func.sig.ident.clone(); + let varident = quote::format_ident!("ITEST_{}", fident); + let fidentstrbuf = format!(r#"{}"#, fident); + let fidentstr = syn::LitStr::new(&fidentstrbuf, Span::call_site()); + let output = quote! { + #[linkme::distributed_slice(TESTS)] + #[allow(non_upper_case_globals)] + static #varident : Test = Test { + name: #fidentstr, + f: #fident, + }; + #func + }; + output.into() +} diff --git a/tests/inst/src/insttest.rs b/tests/inst/src/insttest.rs new file mode 100644 index 00000000..1c1fa379 --- /dev/null +++ b/tests/inst/src/insttest.rs @@ -0,0 +1,46 @@ +use anyhow::Result; +// use structopt::StructOpt; +// // https://github.com/clap-rs/clap/pull/1397 +// #[macro_use] +// extern crate clap; + +mod repobin; +mod sysroot; +mod test; + +fn gather_tests() -> Vec { + test::TESTS + .iter() + .map(|t| libtest_mimic::Test { + name: t.name.into(), + kind: "".into(), + is_ignored: false, + is_bench: false, + data: t, + }) + .collect() +} + +fn run_test(test: &test::TestImpl) -> libtest_mimic::Outcome { + if let Err(e) = (test.data.f)() { + libtest_mimic::Outcome::Failed { + msg: Some(e.to_string()), + } + } else { + libtest_mimic::Outcome::Passed + } +} + +fn main() -> Result<()> { + procspawn::init(); + + // Ensure we're always in tempdir so we can rely on it globally + let tmp_dir = tempfile::Builder::new() + .prefix("ostree-insttest-top") + .tempdir()?; + std::env::set_current_dir(tmp_dir.path())?; + + let args = libtest_mimic::Arguments::from_args(); + let tests = gather_tests(); + libtest_mimic::run_tests(&args, tests, run_test).exit(); +} diff --git a/tests/inst/src/repobin.rs b/tests/inst/src/repobin.rs new file mode 100644 index 00000000..f45f913b --- /dev/null +++ b/tests/inst/src/repobin.rs @@ -0,0 +1,121 @@ +//! Tests that mostly use the CLI and operate on temporary +//! repositories. + +use std::path::Path; + +use crate::test::*; +use anyhow::{Context, Result}; +use commandspec::{sh_command, sh_execute}; +use tokio::runtime::Runtime; +use with_procspawn_tempdir::with_procspawn_tempdir; + +#[itest] +fn test_basic() -> Result<()> { + sh_execute!(r"ostree --help >/dev/null")?; + Ok(()) +} + +#[itest] +#[with_procspawn_tempdir] +fn test_nofifo() -> Result<()> { + assert!(std::path::Path::new(".procspawn-tmpdir").exists()); + sh_execute!( + r"ostree --repo=repo init --mode=archive + mkdir tmproot + mkfifo tmproot/afile +" + )?; + cmd_fails_with( + sh_command!( + r#"ostree --repo=repo commit -b fifotest -s "commit fifo" --tree=dir=./tmproot"# + ) + .unwrap(), + "Not a regular file or symlink", + )?; + Ok(()) +} + +#[itest] +#[with_procspawn_tempdir] +fn test_mtime() -> Result<()> { + sh_execute!( + r"ostree --repo=repo init --mode=archive + mkdir tmproot + echo afile > tmproot/afile + ostree --repo=repo commit -b test --tree=dir=tmproot >/dev/null +" + )?; + let ts = Path::new("repo").metadata()?.modified().unwrap(); + sh_execute!( + r#"ostree --repo=repo commit -b test -s "bump mtime" --tree=dir=tmproot >/dev/null"# + )?; + assert_ne!(ts, Path::new("repo").metadata()?.modified().unwrap()); + Ok(()) +} + +#[itest] +#[with_procspawn_tempdir] +fn test_extensions() -> Result<()> { + sh_execute!(r"ostree --repo=repo init --mode=bare")?; + assert!(Path::new("repo/extensions").exists()); + Ok(()) +} + +async fn impl_test_pull_basicauth() -> Result<()> { + let opts = TestHttpServerOpts { + basicauth: true, + ..Default::default() + }; + let serverrepo = Path::new("server/repo"); + std::fs::create_dir_all(&serverrepo)?; + let addr = http_server(&serverrepo, opts).await?; + tokio::task::spawn_blocking(move || -> Result<()> { + let baseuri = http::Uri::from_maybe_shared(format!("http://{}/", addr).into_bytes())?; + let unauthuri = + http::Uri::from_maybe_shared(format!("http://unknown:badpw@{}/", addr).into_bytes())?; + let authuri = http::Uri::from_maybe_shared( + format!("http://{}@{}/", TEST_HTTP_BASIC_AUTH, addr).into_bytes(), + )?; + let osroot = Path::new("osroot"); + mkroot(&osroot)?; + sh_execute!( + r#"ostree --repo={serverrepo} init --mode=archive + ostree --repo={serverrepo} commit -b os --tree=dir={osroot} >/dev/null + mkdir client + cd client + ostree --repo=repo init --mode=archive + ostree --repo=repo remote add --set=gpg-verify=false origin-unauth {baseuri} + ostree --repo=repo remote add --set=gpg-verify=false origin-badauth {unauthuri} + ostree --repo=repo remote add --set=gpg-verify=false origin-goodauth {authuri} + "#, + osroot = osroot.to_str(), + serverrepo = serverrepo.to_str(), + baseuri = baseuri.to_string(), + unauthuri = unauthuri.to_string(), + authuri = authuri.to_string() + )?; + for rem in &["unauth", "badauth"] { + cmd_fails_with( + sh_command!( + r#"ostree --repo=client/repo pull origin-{rem} os >/dev/null"#, + rem = *rem + ) + .unwrap(), + "HTTP 403", + ) + .context(rem)?; + } + sh_execute!(r#"ostree --repo=client/repo pull origin-goodauth os >/dev/null"#,)?; + Ok(()) + }) + .await??; + Ok(()) +} + +#[itest] +#[with_procspawn_tempdir] +fn test_pull_basicauth() -> Result<()> { + let mut rt = Runtime::new()?; + rt.block_on(async move { impl_test_pull_basicauth().await })?; + Ok(()) +} diff --git a/tests/inst/src/sysroot.rs b/tests/inst/src/sysroot.rs new file mode 100644 index 00000000..08a3d38f --- /dev/null +++ b/tests/inst/src/sysroot.rs @@ -0,0 +1,33 @@ +//! Tests that mostly use the API and access the booted sysroot read-only. + +use anyhow::Result; +use gio::prelude::*; +use ostree::prelude::*; + +use crate::test::*; + +#[itest] +fn test_sysroot_ro() -> Result<()> { + // TODO add a skipped identifier + if !std::path::Path::new("/run/ostree-booted").exists() { + return Ok(()); + } + let cancellable = Some(gio::Cancellable::new()); + let sysroot = ostree::Sysroot::new_default(); + sysroot.load(cancellable.as_ref())?; + assert!(sysroot.is_booted()); + + let booted = sysroot.get_booted_deployment().expect("booted deployment"); + assert!(!booted.is_staged()); + let repo = sysroot.repo().expect("repo"); + + let csum = booted.get_csum().expect("booted csum"); + let csum = csum.as_str(); + + let (root, rev) = repo.read_commit(csum, cancellable.as_ref())?; + assert_eq!(rev, csum); + let root = root.downcast::().expect("downcast"); + root.ensure_resolved()?; + + Ok(()) +} diff --git a/tests/inst/src/test.rs b/tests/inst/src/test.rs new file mode 100644 index 00000000..9e7d4b41 --- /dev/null +++ b/tests/inst/src/test.rs @@ -0,0 +1,180 @@ +use std::borrow::BorrowMut; +use std::fs::File; +use std::io::prelude::*; +use std::path::Path; +use std::process::Command; + +use anyhow::{bail, Context, Result}; +use linkme::distributed_slice; + +pub use itest_macro::itest; +pub use with_procspawn_tempdir::with_procspawn_tempdir; + +// HTTP Server deps +use futures_util::future; +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Request, Response}; +use hyper_staticfile::Static; + +pub(crate) type TestFn = fn() -> Result<()>; + +#[derive(Debug)] +pub(crate) struct Test { + pub(crate) name: &'static str, + pub(crate) f: TestFn, +} + +pub(crate) type TestImpl = libtest_mimic::Test<&'static Test>; + +#[distributed_slice] +pub(crate) static TESTS: [Test] = [..]; + +/// Run command and assert that its stderr contains pat +pub(crate) fn cmd_fails_with>(mut c: C, pat: &str) -> Result<()> { + let c = c.borrow_mut(); + let o = c.output()?; + if o.status.success() { + bail!("Command {:?} unexpectedly succeeded", c); + } + if !twoway::find_bytes(&o.stderr, pat.as_bytes()).is_some() { + dbg!(String::from_utf8_lossy(&o.stdout)); + dbg!(String::from_utf8_lossy(&o.stderr)); + bail!("Command {:?} stderr did not match: {}", c, pat); + } + Ok(()) +} + +pub(crate) fn write_file>(p: P, buf: &str) -> Result<()> { + let p = p.as_ref(); + let mut f = File::create(p)?; + f.write_all(buf.as_bytes())?; + f.flush()?; + Ok(()) +} + +pub(crate) fn mkroot>(p: P) -> Result<()> { + let p = p.as_ref(); + for v in &["usr/bin", "etc"] { + std::fs::create_dir_all(p.join(v))?; + } + let verpath = p.join("etc/version"); + let v: u32 = if verpath.exists() { + let s = std::fs::read_to_string(&verpath)?; + let v: u32 = s.trim_end().parse()?; + v + 1 + } else { + 0 + }; + write_file(&verpath, &format!("{}", v))?; + write_file(p.join("usr/bin/somebinary"), &format!("somebinary v{}", v))?; + write_file(p.join("etc/someconf"), &format!("someconf v{}", v))?; + write_file(p.join("usr/bin/vmod2"), &format!("somebinary v{}", v % 2))?; + write_file(p.join("usr/bin/vmod3"), &format!("somebinary v{}", v % 3))?; + Ok(()) +} + +#[derive(Default, Debug, Copy, Clone)] +pub(crate) struct TestHttpServerOpts { + pub(crate) basicauth: bool, +} + +pub(crate) const TEST_HTTP_BASIC_AUTH: &'static str = "foouser:barpw"; + +fn validate_authz(value: &[u8]) -> Result { + let buf = std::str::from_utf8(&value)?; + if let Some(o) = buf.find("Basic ") { + let (_, buf) = buf.split_at(o + "Basic ".len()); + let buf = base64::decode(buf).context("decoding")?; + let buf = std::str::from_utf8(&buf)?; + Ok(buf == TEST_HTTP_BASIC_AUTH) + } else { + bail!("Missing Basic") + } +} + +pub(crate) async fn http_server>( + p: P, + opts: TestHttpServerOpts, +) -> Result { + let addr = ([127, 0, 0, 1], 0).into(); + let sv = Static::new(p.as_ref()); + + async fn handle_request( + req: Request, + sv: Static, + opts: TestHttpServerOpts, + ) -> Result> { + if opts.basicauth { + if let Some(ref authz) = req.headers().get(http::header::AUTHORIZATION) { + match validate_authz(authz.as_ref()) { + Ok(true) => { + return Ok(sv.clone().serve(req).await?); + } + Ok(false) => { + // Fall through + } + Err(e) => { + return Ok(Response::builder() + .status(hyper::StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(e.to_string())) + .unwrap()); + } + } + }; + return Ok(Response::builder() + .status(hyper::StatusCode::FORBIDDEN) + .header("x-test-auth", "true") + .body(Body::from("not authorized\n")) + .unwrap()); + } + Ok(sv.clone().serve(req).await?) + } + + let make_service = make_service_fn(move |_| { + let sv = sv.clone(); + let opts = opts.clone(); + future::ok::<_, hyper::Error>(service_fn(move |req| handle_request(req, sv.clone(), opts))) + }); + let server: hyper::Server<_, _, _> = hyper::Server::bind(&addr).serve(make_service); + let addr = server.local_addr(); + tokio::spawn(async move { + let r = server.await; + dbg!("server finished!"); + r + }); + Ok(addr) +} + +// I put tests in your tests so you can test while you test +#[cfg(test)] +mod tests { + use super::*; + + fn oops() -> Command { + let mut c = Command::new("/bin/bash"); + c.args(&["-c", "echo oops 1>&2; exit 1"]); + c + } + + #[test] + fn test_fails_with_matches() -> Result<()> { + cmd_fails_with(Command::new("false"), "")?; + cmd_fails_with(oops(), "oops")?; + Ok(()) + } + + #[test] + fn test_fails_with_fails() { + cmd_fails_with(Command::new("true"), "somepat").expect_err("true"); + cmd_fails_with(oops(), "nomatch").expect_err("nomatch"); + } + + #[test] + fn test_validate_authz() -> Result<()> { + assert!(validate_authz("Basic Zm9vdXNlcjpiYXJwdw==".as_bytes())?); + assert!(!validate_authz("Basic dW5rbm93bjpiYWRwdw==".as_bytes())?); + assert!(validate_authz("Basic oops".as_bytes()).is_err()); + assert!(validate_authz("oops".as_bytes()).is_err()); + Ok(()) + } +} diff --git a/tests/kola/nondestructive/.gitignore b/tests/kola/nondestructive/.gitignore new file mode 100644 index 00000000..e2a0c38a --- /dev/null +++ b/tests/kola/nondestructive/.gitignore @@ -0,0 +1,2 @@ +# Generated by runkola.sh +insttest-rs diff --git a/tests/kolainst/Makefile b/tests/kolainst/Makefile index 18305a2f..6416217e 100644 --- a/tests/kolainst/Makefile +++ b/tests/kolainst/Makefile @@ -7,7 +7,9 @@ KOLA_TESTDIR = $(DESTDIR)/usr/lib/coreos-assembler/tests/kola/ostree/ all: for x in $(LIBSCRIPTS); do bash -n "$${x}"; done + (cd ../inst && cargo build --release) -install: +install: all install -D -m 0644 -t $(KOLA_TESTDIR) $(LIBSCRIPTS) for x in $(TESTDIRS); do rsync -rlv ./$${x} $(KOLA_TESTDIR)/; done + install -D -m 0755 -t $(KOLA_TESTDIR)/nondestructive-rs ../inst/target/release/ostree-test diff --git a/tests/runkola b/tests/runkola new file mode 100755 index 00000000..570d7521 --- /dev/null +++ b/tests/runkola @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail +# Generate a new qemu image and run tests +top=$(git rev-parse --show-toplevel) +cd ${top} +make +cosa build-fast +image=$(ls fastbuild-*-qemu.qcow2 | head -1) +if [ -z "${image}" ]; then + echo "failed to find image"; exit 1 +fi +if [ -z "$@" ]; then + set -- 'ext.ostree.*' "$@" +fi +set -x +make -C tests/kolainst +sudo make -C tests/kolainst install +exec kola run -p qemu --qemu-image "${image}" -E ${top} "$@" From 68ebf743cd826586ceccb40d8d1bf7d4c35f3e5f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 28 May 2020 00:41:34 +0000 Subject: [PATCH 146/177] remote-add: Default to explicit sign-verify backends In https://github.com/ostreedev/ostree/pull/2092/commits/588f42e8c64183dfa1fbaa08cc92c46b691b23c4 we added a way to add keys for sign types when doing a `remote add`, and in https://github.com/ostreedev/ostree/pull/2105 we extended `sign-verify` to support *limiting* to an explicit set. This PR changes the *default* for `remote add` to combine the two - when providing an explicit `--sign-verify=type`, we now limit the accepted types to only those. --- src/ostree/ot-remote-builtin-add.c | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/ostree/ot-remote-builtin-add.c b/src/ostree/ot-remote-builtin-add.c index a885336a..172625d2 100644 --- a/src/ostree/ot-remote-builtin-add.c +++ b/src/ostree/ot-remote-builtin-add.c @@ -59,7 +59,7 @@ static GOptionEntry option_entries[] = { { NULL } }; -static gboolean +static char * add_verify_opt (GVariantBuilder *builder, const char *keyspec, GError **error) @@ -68,11 +68,11 @@ add_verify_opt (GVariantBuilder *builder, g_assert (parts && *parts); const char *keytype = parts[0]; if (!parts[1]) - return glnx_throw (error, "Failed to parse KEYTYPE=[inline|file]:DATA in %s", keyspec); + return glnx_null_throw (error, "Failed to parse KEYTYPE=[inline|file]:DATA in %s", keyspec); g_autoptr(OstreeSign) sign = ostree_sign_get_by_name (keytype, error); if (!sign) - return FALSE; + return NULL; const char *rest = parts[1]; g_assert (!parts[2]); @@ -86,13 +86,13 @@ add_verify_opt (GVariantBuilder *builder, else if (g_str_equal (keyref, "file")) optname = g_strdup_printf ("verification-%s-file", keytype); else - return glnx_throw (error, "Invalid key reference %s, expected inline|file", keyref); + return glnx_null_throw (error, "Invalid key reference %s, expected inline|file", keyref); g_assert (keyparts[1] && !keyparts[2]); g_variant_builder_add (builder, "{s@v}", optname, g_variant_new_variant (g_variant_new_string (keyparts[1]))); - return TRUE; + return g_strdup (ostree_sign_get_name (sign)); } gboolean @@ -101,6 +101,7 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio g_autoptr(GOptionContext) context = NULL; g_autoptr(OstreeSysroot) sysroot = NULL; g_autoptr(OstreeRepo) repo = NULL; + g_autoptr(GString) sign_verify = NULL; const char *remote_name; const char *remote_url; char **iter; @@ -193,13 +194,23 @@ ot_remote_builtin_add (int argc, char **argv, OstreeCommandInvocation *invocatio for (char **iter = opt_sign_verify; iter && *iter; iter++) { const char *keyspec = *iter; - if (!add_verify_opt (optbuilder, keyspec, error)) + g_autofree char *signname = add_verify_opt (optbuilder, keyspec, error); + if (!signname) return FALSE; + if (!sign_verify) + { + sign_verify = g_string_new (signname); + } + else + { + g_string_append_c (sign_verify, ','); + g_string_append (sign_verify, signname); + } } - if (opt_sign_verify) + if (sign_verify != NULL) g_variant_builder_add (optbuilder, "{s@v}", "sign-verify", - g_variant_new_variant (g_variant_new_boolean (TRUE))); + g_variant_new_variant (g_variant_new_string (sign_verify->str))); if (opt_collection_id != NULL) g_variant_builder_add (optbuilder, "{s@v}", "collection-id", From 167edbe63a4f9222b49e2eed9d431f7f20c4479d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 29 May 2020 12:31:23 +0000 Subject: [PATCH 147/177] pull: Add error prefixing with specific object when parsing One OpenShift user saw this from rpm-ostree: ``` client(id:cli dbus:1.583 unit:machine-config-daemon-host.service uid:0) added; new total=1 Initiated txn UpdateDeployment for client(id:cli dbus:1.583 unit:machine-config-daemon-host.service uid:0): /org/projectatomic/rpmostree1/rhcos Txn UpdateDeployment on /org/projectatomic/rpmostree1/rhcos failed: File header size 4294967295 exceeds size 0 ``` which isn't very helpful. Let's add some error prefixing here which would at least tell us which object was corrupted. --- src/libostree/ostree-repo-pull.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5a57bfa6..fbcfc8a6 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -999,7 +999,10 @@ content_fetch_on_complete (GObject *object, if (!ostree_content_stream_parse (TRUE, tmpf_input, stbuf.st_size, FALSE, &file_in, &file_info, &xattrs, cancellable, error)) - goto out; + { + g_prefix_error (error, "Parsing %s: ", checksum_obj); + goto out; + } if (verifying_bareuseronly) { From 5f08649f51fe15dcbf034b408a0770d3de3c9658 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Tue, 4 Feb 2020 13:39:27 +0100 Subject: [PATCH 148/177] deploy: support devicetree directory Add support for a devicetree directory at /usr/lib/modules/$kver/dtb/. In ARM world a general purpose distribution often suppports multiple boards with a single operating system. However, OSTree currently only supports a single device tree, which does not allow to use the same OSTree on different ARM machines. In this scenario typically the boot loader selects the effective device tree. This adds device tree directory support for the new boot artefact location under /usr/lib/modules. If the file `devicetree` does not exist, then the folder dtb will be checked. All devicetrees are hashed into the deployment hash. This makes sure that even a single devicetree change leads to a new deployment and hence can be rolled back. The loader configuration has a new key "devicetreepath" which contains the path where devicetrees are stored. This is also written to the U-Boot variable "fdtdir". The boot loader is expected to use this path to load a particular machines device tree from. Closes: #1900 Signed-off-by: Stefan Agner --- src/libostree/ostree-bootloader-uboot.c | 4 + src/libostree/ostree-sysroot-deploy.c | 127 ++++++++++++++++++++++-- tests/test-admin-deploy-uboot.sh | 30 +++++- 3 files changed, 151 insertions(+), 10 deletions(-) diff --git a/src/libostree/ostree-bootloader-uboot.c b/src/libostree/ostree-bootloader-uboot.c index 4cd955d5..1e1f0371 100644 --- a/src/libostree/ostree-bootloader-uboot.c +++ b/src/libostree/ostree-bootloader-uboot.c @@ -144,6 +144,10 @@ create_config_from_boot_loader_entries (OstreeBootloaderUboot *self, if (val) g_ptr_array_add (new_lines, g_strdup_printf ("fdt_file%s=%s", index_suffix, val)); + val = ostree_bootconfig_parser_get (config, "fdtdir"); + if (val) + g_ptr_array_add (new_lines, g_strdup_printf ("fdtdir%s=%s", index_suffix, val)); + val = ostree_bootconfig_parser_get (config, "options"); if (val) { diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index ee00c02c..cb593020 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -217,6 +217,79 @@ dirfd_copy_attributes_and_xattrs (int src_parent_dfd, return TRUE; } +static gint +str_sort_cb (gconstpointer name_ptr_a, gconstpointer name_ptr_b) +{ + const gchar *name_a = *((const gchar **) name_ptr_a); + const gchar *name_b = *((const gchar **) name_ptr_b); + + return g_strcmp0 (name_a, name_b); +} + +static gboolean +checksum_dir_recurse (int dfd, + const char *path, + OtChecksum *checksum, + GCancellable *cancellable, + GError **error) +{ + g_auto(GLnxDirFdIterator) dfditer = { 0, }; + g_autoptr (GPtrArray) d_entries = g_ptr_array_new_with_free_func (g_free); + + if (!glnx_dirfd_iterator_init_at (dfd, path, TRUE, &dfditer, error)) + return FALSE; + + while (TRUE) + { + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent (&dfditer, &dent, cancellable, error)) + return FALSE; + + if (dent == NULL) + break; + + g_ptr_array_add (d_entries, g_strdup (dent->d_name)); + } + + /* File systems do not guarantee dir entry order, make sure this is + * reproducable + */ + g_ptr_array_sort(d_entries, str_sort_cb); + + for (gint i=0; i < d_entries->len; i++) + { + const gchar *d_name = (gchar *)g_ptr_array_index (d_entries, i); + struct stat stbuf; + + if (!glnx_fstatat (dfditer.fd, d_name, &stbuf, + AT_SYMLINK_NOFOLLOW, error)) + return FALSE; + + if (S_ISDIR (stbuf.st_mode)) + { + if (!checksum_dir_recurse(dfditer.fd, d_name, checksum, cancellable, error)) + return FALSE; + } + else + { + int fd; + + if (!ot_openat_ignore_enoent (dfditer.fd, d_name, &fd, error)) + return FALSE; + if (fd != -1) + { + g_autoptr(GInputStream) in = g_unix_input_stream_new (fd, FALSE); + if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error)) + return FALSE; + } + } + + } + + return TRUE; +} + static gboolean copy_dir_recurse (int src_parent_dfd, int dest_parent_dfd, @@ -1065,6 +1138,9 @@ get_kernel_from_tree_usrlib_modules (int deployment_dfd, g_clear_object (&in); glnx_close_fd (&fd); + /* Check for /usr/lib/modules/$kver/devicetree first, if it does not + * exist check for /usr/lib/modules/$kver/dtb/ directory. + */ if (!ot_openat_ignore_enoent (ret_layout->boot_dfd, "devicetree", &fd, error)) return FALSE; if (fd != -1) @@ -1075,6 +1151,23 @@ get_kernel_from_tree_usrlib_modules (int deployment_dfd, if (!ot_gio_splice_update_checksum (NULL, in, &checksum, cancellable, error)) return FALSE; } + else + { + struct stat stbuf; + /* Check for dtb directory */ + if (!glnx_fstatat_allow_noent (ret_layout->boot_dfd, "dtb", &stbuf, 0, error)) + return FALSE; + + if (errno == 0 && S_ISDIR (stbuf.st_mode)) + { + /* devicetree_namever set to NULL indicates a complete directory */ + ret_layout->devicetree_srcpath = g_strdup ("dtb"); + ret_layout->devicetree_namever = NULL; + + if (!checksum_dir_recurse(ret_layout->boot_dfd, "dtb", &checksum, cancellable, error)) + return FALSE; + } + } g_clear_object (&in); glnx_close_fd (&fd); @@ -1730,15 +1823,24 @@ install_deployment_kernel (OstreeSysroot *sysroot, if (kernel_layout->devicetree_srcpath) { - g_assert (kernel_layout->devicetree_namever); - if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->devicetree_namever, &stbuf, 0, error)) - return FALSE; - if (errno == ENOENT) + /* If devicetree_namever is set a single device tree is deployed */ + if (kernel_layout->devicetree_namever) { - if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->devicetree_srcpath, - bootcsum_dfd, kernel_layout->devicetree_namever, - sysroot->debug_flags, - cancellable, error)) + if (!glnx_fstatat_allow_noent (bootcsum_dfd, kernel_layout->devicetree_namever, &stbuf, 0, error)) + return FALSE; + if (errno == ENOENT) + { + if (!install_into_boot (repo, sepolicy, kernel_layout->boot_dfd, kernel_layout->devicetree_srcpath, + bootcsum_dfd, kernel_layout->devicetree_namever, + sysroot->debug_flags, + cancellable, error)) + return FALSE; + } + } + else + { + if (!copy_dir_recurse(kernel_layout->boot_dfd, bootcsum_dfd, kernel_layout->devicetree_srcpath, + sysroot->debug_flags, cancellable, error)) return FALSE; } } @@ -1850,6 +1952,15 @@ install_deployment_kernel (OstreeSysroot *sysroot, g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_namever, NULL); ostree_bootconfig_parser_set (bootconfig, "devicetree", boot_relpath); } + else if (kernel_layout->devicetree_srcpath) + { + /* If devicetree_srcpath is set but devicetree_namever is NULL, then we + * want to point to a whole directory of device trees. + * See: https://github.com/ostreedev/ostree/issues/1900 + */ + g_autofree char * boot_relpath = g_strconcat ("/", bootcsumdir, "/", kernel_layout->devicetree_srcpath, NULL); + ostree_bootconfig_parser_set (bootconfig, "fdtdir", boot_relpath); + } /* Note this is parsed in ostree-impl-system-generator.c */ g_autofree char *ostree_kernel_arg = g_strdup_printf ("ostree=/ostree/boot.%d/%s/%s/%d", diff --git a/tests/test-admin-deploy-uboot.sh b/tests/test-admin-deploy-uboot.sh index 8ea37fe9..e3163cb0 100755 --- a/tests/test-admin-deploy-uboot.sh +++ b/tests/test-admin-deploy-uboot.sh @@ -25,9 +25,11 @@ set -euo pipefail . $(dirname $0)/libtest.sh # Exports OSTREE_SYSROOT so --sysroot not needed. -setup_os_repository "archive" "uboot" +kver="3.6.0" +modulesdir="usr/lib/modules/${kver}" +setup_os_repository "archive" "uboot" ${modulesdir} -extra_admin_tests=1 +extra_admin_tests=2 . $(dirname $0)/admin-test.sh @@ -52,3 +54,27 @@ assert_file_has_content sysroot/boot/uEnv.txt "kernel_image2=" assert_file_has_content sysroot/boot/uEnv.txt "kernel_image3=" echo "ok merging uEnv.txt files" + +cd ${test_tmpdir} +os_repository_new_commit "uboot test" "test with device tree directory" + +devicetree_path=osdata/${modulesdir}/dtb/asoc-board.dtb +devicetree_overlay_path=osdata/${modulesdir}/dtb/overlays/overlay.dtbo + +mkdir -p osdata/${modulesdir}/dtb +echo "a device tree" > ${devicetree_path} +mkdir -p osdata/${modulesdir}/dtb/overlays +echo "a device tree overlay" > ${devicetree_overlay_path} + +bootcsum=$( + (echo "new: a kernel uboot test" && echo "new: an initramfs uboot test" && + cat ${devicetree_path} ${devicetree_overlay_path} ) | + sha256sum | cut -f 1 -d ' ') + +${CMD_PREFIX} ostree --repo=testos-repo commit --tree=dir=osdata/ -b testos/buildmaster/x86_64-runtime +${CMD_PREFIX} ostree admin upgrade --os=testos +assert_file_has_content sysroot/boot/uEnv.txt "fdtdir=" +assert_file_has_content sysroot/boot/ostree/testos-${bootcsum}/dtb/asoc-board.dtb 'a device tree' +assert_file_has_content sysroot/boot/ostree/testos-${bootcsum}/dtb/overlays/overlay.dtbo 'a device tree overlay' + +echo "ok deploying fdtdir" From 97cda7ff4dd2a705cf44dd6ec90162ef43f619fc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 31 May 2020 17:58:44 +0000 Subject: [PATCH 149/177] bupsplit: rustfmt(*) Let's use the standard rustfmt style. Also remove unused parenthesis which rust-analyzer was complaining about. Also add a `.gitignore`. --- rust/.gitignore | 2 ++ rust/src/bupsplit.rs | 40 ++++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 rust/.gitignore diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 00000000..1e7caa9e --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +target/ diff --git a/rust/src/bupsplit.rs b/rust/src/bupsplit.rs index 607c6639..73dc9ee1 100644 --- a/rust/src/bupsplit.rs +++ b/rust/src/bupsplit.rs @@ -43,9 +43,9 @@ const ROLLSUM_CHAR_OFFSET: u32 = 31; // Previously in the header file const BUP_BLOBBITS: u32 = 13; -const BUP_BLOBSIZE: u32 = (1< Rollsum { - Rollsum { s1: BUP_WINDOWSIZE * ROLLSUM_CHAR_OFFSET, - s2: BUP_WINDOWSIZE * (BUP_WINDOWSIZE-1) * ROLLSUM_CHAR_OFFSET, - window: [0; 64], - wofs: 0 + Rollsum { + s1: BUP_WINDOWSIZE * ROLLSUM_CHAR_OFFSET, + s2: BUP_WINDOWSIZE * (BUP_WINDOWSIZE - 1) * ROLLSUM_CHAR_OFFSET, + window: [0; 64], + wofs: 0, } } @@ -67,8 +68,13 @@ impl Rollsum { pub fn add(&mut self, drop: u8, add: u8) -> () { let drop_expanded = u32::from(drop); let add_expanded = u32::from(add); - self.s1 = self.s1.wrapping_add(add_expanded.wrapping_sub(drop_expanded)); - self.s2 = self.s2.wrapping_add(self.s1.wrapping_sub(BUP_WINDOWSIZE * (drop_expanded + ROLLSUM_CHAR_OFFSET))); + self.s1 = self + .s1 + .wrapping_add(add_expanded.wrapping_sub(drop_expanded)); + self.s2 = self.s2.wrapping_add( + self.s1 + .wrapping_sub(BUP_WINDOWSIZE * (drop_expanded + ROLLSUM_CHAR_OFFSET)), + ); } pub fn roll(&mut self, ch: u8) -> () { @@ -85,7 +91,7 @@ impl Rollsum { } #[no_mangle] -pub extern fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) -> u32 { +pub extern "C" fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) -> u32 { let sbuf = unsafe { assert!(!buf.is_null()); slice::from_raw_parts(buf.offset(ofs as isize), (len - ofs) as usize) @@ -99,9 +105,11 @@ pub extern fn bupsplit_sum(buf: *const u8, ofs: libc::size_t, len: libc::size_t) } #[no_mangle] -pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, - bits: *mut libc::c_int) -> libc::c_int -{ +pub extern "C" fn bupsplit_find_ofs( + buf: *const u8, + len: libc::size_t, + bits: *mut libc::c_int, +) -> libc::c_int { if buf.is_null() { return 0; } @@ -110,10 +118,10 @@ pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, let mut r = Rollsum::new(); for x in sbuf { r.roll(*x); - if (r.s2 & (BUP_BLOBSIZE-1)) == ((u32::max_value()) & (BUP_BLOBSIZE-1)) { - if !bits.is_null() { + if (r.s2 & (BUP_BLOBSIZE - 1)) == ((u32::max_value()) & (BUP_BLOBSIZE - 1)) { + if !bits.is_null() { let mut sum = r.digest() >> BUP_BLOBBITS; - let mut rbits : libc::c_int = BUP_BLOBBITS as i32; + let mut rbits: libc::c_int = BUP_BLOBBITS as i32; while sum & 1 != 0 { sum >>= 1; rbits += 1; @@ -122,7 +130,7 @@ pub extern fn bupsplit_find_ofs(buf: *const u8, len: libc::size_t, *bits = rbits; } } - return len as i32 + return len as i32; } } 0 From 1cd902cd1a0f9c4aaa325b45aaba961567e35448 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 4 Jun 2020 12:24:16 +0000 Subject: [PATCH 150/177] tests/rust: Extract a with_webserver_in helper wrapper It's much cleaner if the Tokio stuff stays in `test.rs`, and easier to write tests if the function is synchronous. Prep for further tests. --- tests/inst/src/repobin.rs | 19 +++++-------------- tests/inst/src/test.rs | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/inst/src/repobin.rs b/tests/inst/src/repobin.rs index f45f913b..41fd1390 100644 --- a/tests/inst/src/repobin.rs +++ b/tests/inst/src/repobin.rs @@ -6,7 +6,6 @@ use std::path::Path; use crate::test::*; use anyhow::{Context, Result}; use commandspec::{sh_command, sh_execute}; -use tokio::runtime::Runtime; use with_procspawn_tempdir::with_procspawn_tempdir; #[itest] @@ -61,15 +60,16 @@ fn test_extensions() -> Result<()> { Ok(()) } -async fn impl_test_pull_basicauth() -> Result<()> { +#[itest] +#[with_procspawn_tempdir] +fn test_pull_basicauth() -> Result<()> { let opts = TestHttpServerOpts { basicauth: true, ..Default::default() }; let serverrepo = Path::new("server/repo"); std::fs::create_dir_all(&serverrepo)?; - let addr = http_server(&serverrepo, opts).await?; - tokio::task::spawn_blocking(move || -> Result<()> { + with_webserver_in(&serverrepo, &opts, move |addr| { let baseuri = http::Uri::from_maybe_shared(format!("http://{}/", addr).into_bytes())?; let unauthuri = http::Uri::from_maybe_shared(format!("http://unknown:badpw@{}/", addr).into_bytes())?; @@ -107,15 +107,6 @@ async fn impl_test_pull_basicauth() -> Result<()> { } sh_execute!(r#"ostree --repo=client/repo pull origin-goodauth os >/dev/null"#,)?; Ok(()) - }) - .await??; - Ok(()) -} - -#[itest] -#[with_procspawn_tempdir] -fn test_pull_basicauth() -> Result<()> { - let mut rt = Runtime::new()?; - rt.block_on(async move { impl_test_pull_basicauth().await })?; + })?; Ok(()) } diff --git a/tests/inst/src/test.rs b/tests/inst/src/test.rs index 9e7d4b41..7178d7bb 100644 --- a/tests/inst/src/test.rs +++ b/tests/inst/src/test.rs @@ -15,6 +15,7 @@ use futures_util::future; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response}; use hyper_staticfile::Static; +use tokio::runtime::Runtime; pub(crate) type TestFn = fn() -> Result<()>; @@ -145,6 +146,23 @@ pub(crate) async fn http_server>( Ok(addr) } +pub(crate) fn with_webserver_in, F>( + path: P, + opts: &TestHttpServerOpts, + f: F) -> Result<()> +where + F: FnOnce(&std::net::SocketAddr) -> Result<()>, + F: Send + 'static, +{ + let path = path.as_ref(); + let mut rt = Runtime::new()?; + rt.block_on(async move { + let addr = http_server(path, opts.clone()).await?; + tokio::task::spawn_blocking(move || f(&addr)).await? + })?; + Ok(()) +} + // I put tests in your tests so you can test while you test #[cfg(test)] mod tests { From b1378950229b6d9a1b4d485ca6d287d301c59579 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 4 Jun 2020 21:06:52 +0000 Subject: [PATCH 151/177] commit: Note in help that --base takes an argument I was trying to use this in some testing work and was confused for a minute. --- src/ostree/ot-builtin-commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 253a58f7..20409fc2 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -103,7 +103,7 @@ static GOptionEntry options[] = { { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, { "no-bindings", 0, 0, G_OPTION_ARG_NONE, &opt_no_bindings, "Do not write any ref bindings", NULL }, { "bind-ref", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_bind_refs, "Add a ref to ref binding commit metadata", "BRANCH" }, - { "base", 0, 0, G_OPTION_ARG_STRING, &opt_base, "Start from the given commit as a base (no modifiers apply)" }, + { "base", 0, 0, G_OPTION_ARG_STRING, &opt_base, "Start from the given commit as a base (no modifiers apply)", "REF" }, { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "dir=PATH or tar=TARFILE or ref=COMMIT" }, { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Add a key/value pair to metadata", "KEY=VALUE" }, { "add-metadata", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_variants, "Add a key/value pair to metadata, where the KEY is a string, an VALUE is g_variant_parse() formatted", "KEY=VALUE" }, From 29dddf38d35792024c0b124727c6d10771134170 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 3 Jun 2020 18:52:34 +0000 Subject: [PATCH 152/177] core: Add OSTREE_COMMIT_META_KEY_ARCHITECTURE Add a standard key for this. We actually had a case in OpenShift builds recently where a `ppc64le` image was pushed over an `x86_64` one and this started failing at runtime with a not immediately obvious error. I'll probably end up changing rpm-ostree at least to use the RPM architecture for this key and fail if it doesn't match the booted value. Possibly that should live in ostree but it would involve adding architecture schema here, which gets into a big mess. Let's just standardize the key. xref https://github.com/coreos/coreos-assembler/commit/e02ef2683d688607e7b5ad9ea6a0c00c50a682a5 --- src/libostree/ostree-core.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 540f3232..3b903d5c 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -212,6 +212,19 @@ typedef enum { * Since: 2014.9 */ #define OSTREE_COMMIT_META_KEY_VERSION "version" + +/** + * OSTREE_COMMIT_META_KEY_ARCHITECTURE: + * + * GVariant type `s`. Intended to describe the CPU architecture. This is a freeform string, and some distributions + * which have existing package managers might want to match that schema. If you + * don't have a prior schema, it's recommended to use `uname -m` by default (i.e. the Linux kernel schema). In the future + * ostree might include a builtin function to compare architectures. + * + * Since: 2020.4 + */ +#define OSTREE_COMMIT_META_KEY_ARCHITECTURE "ostree.architecture" + /** * OSTREE_COMMIT_META_KEY_ENDOFLIFE_REBASE: * From 843482e589e3b989e88d892e6d1fca002a5887d2 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Mon, 8 Jun 2020 15:17:58 +0200 Subject: [PATCH 153/177] man/checkout: fix short name option of --user-mode The short name option of --user-mode is -U. Signed-off-by: Stefan Agner --- man/ostree-checkout.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml index 06385927..8ba1c249 100644 --- a/man/ostree-checkout.xml +++ b/man/ostree-checkout.xml @@ -68,7 +68,7 @@ Boston, MA 02111-1307, USA. - , + , Do not change file ownership or initialize extended attributes. From 832ca09891405f9079124cc9e76941531bb54630 Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Mon, 8 Jun 2020 15:40:52 +0200 Subject: [PATCH 154/177] checkout: use FILE as option argument string for --skip-list Align with --from-file and use 'FILE' instead of 'PATH' as option argument string. No functional change, this is only cosmetics. Signed-off-by: Stefan Agner --- src/ostree/ot-builtin-checkout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index a0fe4be4..813dbb9e 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -87,7 +87,7 @@ static GOptionEntry options[] = { { "force-copy-zerosized", 'z', 0, G_OPTION_ARG_NONE, &opt_force_copy_zerosized, "Do not hardlink zero-sized files", NULL }, { "force-copy", 'C', 0, G_OPTION_ARG_NONE, &opt_force_copy, "Never hardlink (but may reflink if available)", NULL }, { "bareuseronly-dirs", 'M', 0, G_OPTION_ARG_NONE, &opt_bareuseronly_dirs, "Suppress mode bits outside of 0775 for directories (suid, world writable, etc.)", NULL }, - { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" }, + { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "FILE" }, { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /); implies --force-copy", "PATH" }, { "selinux-prefix", 0, 0, G_OPTION_ARG_STRING, &opt_selinux_prefix, "When setting SELinux labels, prefix all paths by PREFIX", "PREFIX" }, { NULL } From 38aa912a0eff7f555ef3cd05c76edc3641c02dac Mon Sep 17 00:00:00 2001 From: Stefan Agner Date: Mon, 8 Jun 2020 15:42:38 +0200 Subject: [PATCH 155/177] man/checkout: document missing options Document missing options in the ostree checkout man page. Signed-off-by: Stefan Agner --- man/ostree-checkout.xml | 71 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml index 8ba1c249..3956e34f 100644 --- a/man/ostree-checkout.xml +++ b/man/ostree-checkout.xml @@ -108,6 +108,14 @@ Boston, MA 02111-1307, USA. directories. Requires -H. + + + + + Process whiteout files (Docker style). + + + @@ -131,6 +139,69 @@ Boston, MA 02111-1307, USA. Process many checkouts from input file. + + + ="POLICY" + + + POLICY is a boolean which specifies whether fsync should be + used or not. Default to true. + + + + + , + + + + Do not fall back to full copies if hardlinking fails. + + + + + , + + + + Do not hardlink zero-sized files. + + + + + , + + + Never hardlink (but may reflink if available). + + + + + , + + + + Suppress mode bits outside of 0775 for directories (suid, + world writable, etc.). + + + + + ="FILE" + + + Skip checking out the absolute file paths listed in FILE, + one per line. + + + + + + + + Set SELinux labels based on policy in root filesystem PATH + (may be /). This implies --force-copy. + + From b82c296198e48e1bfba76b5f7716e97b0ef2fbf0 Mon Sep 17 00:00:00 2001 From: NEPO <54682807+zpiotr@users.noreply.github.com> Date: Sat, 9 May 2020 12:23:50 +0200 Subject: [PATCH 156/177] README.md: Fix link to CONTRIBUTING.md We should link to the target and not the symlink. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e6348dd..1ef8e302 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs. Contributing ------------ -See [Contributing](CONTRIBUTING.md). +See [Contributing](docs/CONTRIBUTING.md). Licensing From 40d6f6b5eee9d7bb5e29663eddbf659ca0818a73 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 11 Jun 2020 18:31:33 +0000 Subject: [PATCH 157/177] tests: Add a pre-signed-pull.sh test I'm thinking about adding an implementation of ed25519 signatures with OpenSSL (so we can ship the feature with Fedora CoreOS without requiring an additional library) and in preparation for that it's essential that we validate that libsodium-generated signatures and OpenSSL-generated signatures are compatible. I don't know if they are yet actually, but the goal of this new test is to add a pre-generated repository with a signed commit generated by libsodium. This will catch if e.g. there's ever a change in libsodium, or if existing libsodium implementation versions (e.g. the one in Debian) might differ from what we ship here. --- Makefile-tests.am | 2 ++ tests/pre-signed-pull-data.tar.gz | Bin 0 -> 1633 bytes tests/test-pre-signed-pull.sh | 52 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 tests/pre-signed-pull-data.tar.gz create mode 100755 tests/test-pre-signed-pull.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 411c5628..a4179377 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -140,6 +140,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-config.sh \ tests/test-signed-commit.sh \ tests/test-signed-pull.sh \ + tests/test-pre-signed-pull.sh \ tests/test-signed-pull-summary.sh \ $(NULL) @@ -201,6 +202,7 @@ dist_installed_test_data = tests/archive-test.sh \ tests/fah-deltadata-old.tar.xz \ tests/fah-deltadata-new.tar.xz \ tests/ostree-path-traverse.tar.gz \ + tests/pre-signed-pull-data.tar.gz \ tests/libtest-core.sh \ $(NULL) diff --git a/tests/pre-signed-pull-data.tar.gz b/tests/pre-signed-pull-data.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..53a6019abe143c497b27519406b328f0d8343505 GIT binary patch literal 1633 zcmV-n2A=sJiwFP!000001MQo8XdFcx#}7r&>qDaj@dblmX@zWNW@jHKL~Wa#nx?6F zTxwgb%K{6n zTr>~mNbGK+&WD5DyWN}L?tb_8o!`veY8{2lffCCw8cSpuaPD%J0@6(j5(^U@6(B;0 zm<3>De`tlmSd!8hfWkQSQywROyI-sA|5CJ$8lTi0lVlj9A+Slv-|8<#pVc245KH|r zCRPE!43Fs#{^$Dl4#Ko|5N#W%u%01rYk#HY@9G~OC_mJlmUm5WgXy*!>gwz6+ghoV z^?tv+XP`To91hvGe(0A6E6A^O5BE7!dq#^aV59w0?!>O)9=x+VePYjptj>paV;Apv z*say;JgDxh)OxyEkWTJ^mF>;o<)K>tNTuAAg`KDvM2vUR*sXf|dHWPr7W|*3ng3}}&#n(xhW}g2|MOvv{w{T^ zJd3+v3Hs0azh&x5|2$~w&nHqIHq1ZC;09Q_{zONX(mxNH`pb)uW|I32mXZIMK&5{k zH1&_U$RhJ+Scd-4Q2Cz=bM>!s%2wF_oAn<-a-sCkfyMf#JW1u-D;T5I_#Z*EK!3x) zx*GrIflG+VC^T8AL>T6!k}jwLZHE!aDMwr{VTZX=AqFvZb<5T{bcHO>b=x!@-BnWq z|8C~yf8fP&6kqusU>WtlVJQF4g}M62Jcy+GuL}L;Lg}9aP5mQhl)Gt0Lx82%f4V`i z(mxLt=ugP1sQ-1#QuDvLuvmYSLM{+;EZ0S*C3MGxIyY>?wYbDFEkjUB5M~BG$*X$0o)^)&Kdj<=d4$0!98T$bTqSh5r*s)cjvA zEY_bhj&1DNwjA}BASn^q85ky(!+mB-kubIq;l##5m?g#?3>iX%2!q+S;8+u$&!^_c zSn~W2Hjt(KKQC^bvwp{)t4|-EUfivmpPC(yH)(S^q2l&xOVM+kzR)f{=5PDmYn)-jK&9!=e?Ii_)TJ+8dgQ9&@wW~gKL6rtZ5M95 zrWb7i_XTkK+A~kRUOBp6|MmS3`#vpQcjAM+8;;zZJp0x7@l%g}-q~~Zx+ClEFHkzU zVc!a$ouU=Z{MY=*tz{KsCja45#{W?L{*w<{$3XX%t{vT4z%q-`%KznPf5G$ra-p98 z%Y~t;mjL;qpy30Fmqzg<828eu{A6Cm2S#|vW17kj#u8a37Sk#RBJ%xcT)qR%X9tse zibc>4%A7?p1v&;3w)wb8WE){hJ=p;80y@GQ`^@w_fF>Xk%^3Fpim=Tc*$Yuy2*)Y8 z&?+&*F|kdBQ-T&pL@;QhMgzX*BoXt*f_9`q4cYCwCR`nBGa-w`A$fo!5%ocp$GmZl zX#^T&El8_01tA{?Q4E+TgxnmaAel^39wfk}A=poSkI8eDCn>)t*Qf?SS?wjEP5f3O?sg`$2v_t$Di3t?j3^Z_NNbOJQ|Zl fwxFJ`ssB_e6bgkxp-?CkiY)jYIFj*m08jt`herr.txt; then + fatal "pulled with wrong key" +fi +assert_file_has_content err.txt 'error:.* no valid ed25519 signatures found' +echo "ok pre-signed pull" From 1f3c8c5b3de978f6e069c24938967f823cce7ee8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 Jun 2020 13:18:07 +0000 Subject: [PATCH 158/177] sign/ed25519: Output failed signatures in error message To aid debuggability, when we find a commit that isn't signed by our expected key, output a specific error message with the key. (And then add code to switch to just printing the count beyond 3 because the test suite injects 100 keys and hopefully no one ever actually does that) --- src/libostree/ostree-sign-ed25519.c | 24 +++++++++++++++++++++--- tests/test-pre-signed-pull.sh | 2 +- tests/test-signed-commit.sh | 5 +++-- tests/test-signed-pull-summary.sh | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 4d984d1e..05fbe5eb 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -202,6 +202,9 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, g_debug ("verify: data hash = 0x%x", g_bytes_hash(data)); + g_autoptr(GString) invalid_signatures = NULL; + guint n_invalid_signatures = 0; + for (gsize i = 0; i < g_variant_n_children(signatures); i++) { g_autoptr (GVariant) child = g_variant_get_child_value (signatures, i); @@ -230,8 +233,13 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, public_key->data) != 0) { /* Incorrect signature! */ - g_debug("Signature couldn't be verified with key '%s'", - sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + if (invalid_signatures == NULL) + invalid_signatures = g_string_new (""); + else + g_string_append (invalid_signatures, "; "); + n_invalid_signatures++; + g_string_append_printf (invalid_signatures, "key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); } else { @@ -242,7 +250,17 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, } } - return glnx_throw (error, "no valid ed25519 signatures found"); + if (invalid_signatures) + { + g_assert_cmpuint (n_invalid_signatures, >, 0); + /* The test suite has a key ring with 100 keys. This seems insane, let's + * cap a reasonable error message at 3. + */ + if (n_invalid_signatures > 3) + return glnx_throw (error, "ed25519: Signature couldn't be verified; tried %u keys", n_invalid_signatures); + return glnx_throw (error, "ed25519: Signature couldn't be verified with: %s", invalid_signatures->str); + } + return glnx_throw (error, "ed25519: no signatures found"); #endif /* HAVE_LIBSODIUM */ return FALSE; diff --git a/tests/test-pre-signed-pull.sh b/tests/test-pre-signed-pull.sh index ae4e26f9..20f2b597 100755 --- a/tests/test-pre-signed-pull.sh +++ b/tests/test-pre-signed-pull.sh @@ -48,5 +48,5 @@ ostree --repo=repo remote add badupstream --set=gpg-verify=false --sign-verify=e if ostree --repo=repo pull badupstream:testref 2>err.txt; then fatal "pulled with wrong key" fi -assert_file_has_content err.txt 'error:.* no valid ed25519 signatures found' +assert_file_has_content err.txt 'error:.* ed25519: Signature couldn.t be verified with: key' echo "ok pre-signed pull" diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 4dcf38a4..6bdbfdd6 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -148,9 +148,10 @@ for((i=0;i<100;i++)); do gen_ed25519_random_public done > ${PUBKEYS} # Check if file contain no valid signatures -if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT}; then - exit 1 +if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} 2>err.txt; then + fatal "validated with no signatures" fi +assert_file_has_content err.txt 'error:.* ed25519: Signature couldn.t be verified; tried 100 keys' # Check if no valid signatures provided via args&file if ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} ${WRONG_PUBLIC}; then exit 1 diff --git a/tests/test-signed-pull-summary.sh b/tests/test-signed-pull-summary.sh index 6a240635..e953f2ea 100755 --- a/tests/test-signed-pull-summary.sh +++ b/tests/test-signed-pull-summary.sh @@ -226,7 +226,7 @@ cp ${test_tmpdir}/ostree-srv/gnomerepo/summary.sig{.2,} if ${OSTREE} --repo=repo pull origin main 2>err.txt; then assert_not_reached "Successful pull with old summary" fi -assert_file_has_content err.txt "no valid ed25519 signatures found" +assert_file_has_content err.txt "ed25519: Signature couldn't be verified with: key" assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig cmp repo/tmp/cache/summaries/origin ${test_tmpdir}/ostree-srv/gnomerepo/summary.1 >&2 From b3694b55acc3a700c22acd5c056577194c3bef4e Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 16 Jun 2020 16:30:23 -0400 Subject: [PATCH 159/177] libglnx: Bump to latest For `copy_file_range` fix: https://gitlab.gnome.org/GNOME/libglnx/-/merge_requests/18 Update submodule: libglnx --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 9d0711fe..84b981a2 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 9d0711fe1d6db58b4e24a6c92113930fecafd43e +Subproject commit 84b981a2ef65b46040f7529839448d9219687ac8 From 36258036ae1388c728eec0237b283e225f64a7b1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 17 Jun 2020 00:22:49 +0000 Subject: [PATCH 160/177] signapi: Change API to also return a success message This is the dual of https://github.com/ostreedev/ostree/pull/2129/commits/1f3c8c5b3de978f6e069c24938967f823cce7ee8 where we output more detail when signapi fails to validate. Extend the API to return a string for success, which we output to stdout. This will help the test suite *and* end users validate that the expected thing is happening. In order to make this cleaner, split the "verified commit" set in the pull code into GPG and signapi verified sets, and have the signapi verified set contain the verification string. We're not doing anything with the verification string in the pull code *yet* but I plan to add something like `ostree pull --verbose` which would finally print this. --- src/libostree/ostree-repo-pull-private.h | 2 ++ src/libostree/ostree-repo-pull-verify.c | 21 ++++++++++++++------- src/libostree/ostree-repo-pull.c | 19 +++++++++++++++---- src/libostree/ostree-sign-dummy.c | 7 ++++++- src/libostree/ostree-sign-dummy.h | 1 + src/libostree/ostree-sign-ed25519.c | 9 +++++++-- src/libostree/ostree-sign-ed25519.h | 1 + src/libostree/ostree-sign.c | 5 ++++- src/libostree/ostree-sign.h | 3 +++ src/ostree/ot-builtin-sign.c | 10 +++++++++- tests/test-signed-commit.sh | 9 ++++++--- 11 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index fd17baee..689118be 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -77,6 +77,7 @@ typedef struct { GHashTable *summary_deltas_checksums; GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ GHashTable *verified_commits; /* Set of commits that have been verified */ + GHashTable *signapi_verified_commits; /* Map of commits that have been signapi verified */ GHashTable *ref_keyring_map; /* Maps OstreeCollectionRef to keyring remote name */ GPtrArray *static_delta_superblocks; GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */ @@ -149,6 +150,7 @@ gboolean _sign_verify_for_remote (GPtrArray *signers, GBytes *signed_data, GVariant *metadata, + char **out_success_message, GError **error); gboolean diff --git a/src/libostree/ostree-repo-pull-verify.c b/src/libostree/ostree-repo-pull-verify.c index ab680daf..fa170f94 100644 --- a/src/libostree/ostree-repo-pull-verify.c +++ b/src/libostree/ostree-repo-pull-verify.c @@ -261,12 +261,15 @@ gboolean _sign_verify_for_remote (GPtrArray *verifiers, GBytes *signed_data, GVariant *metadata, + char **out_success_message, GError **error) { guint n_invalid_signatures = 0; g_autoptr (GError) last_sig_error = NULL; gboolean found_sig = FALSE; + g_assert (out_success_message == NULL || *out_success_message == NULL); + g_assert_cmpuint (verifiers->len, >=, 1); for (guint i = 0; i < verifiers->len; i++) { @@ -282,17 +285,21 @@ _sign_verify_for_remote (GPtrArray *verifiers, found_sig = TRUE; + g_autofree char *success_message = NULL; /* Return true if any signature fit to pre-loaded public keys. * If no keys configured -- then system configuration will be used */ if (!ostree_sign_data_verify (sign, signed_data, signatures, + &success_message, last_sig_error ? NULL : &last_sig_error)) { n_invalid_signatures++; continue; } /* Accept the first valid signature */ + if (out_success_message) + *out_success_message = g_steal_pointer (&success_message); return TRUE; } @@ -348,11 +355,10 @@ _verify_unwritten_commit (OtPullData *pull_data, GCancellable *cancellable, GError **error) { - - if (pull_data->gpg_verify || pull_data->signapi_commit_verifiers) - /* Shouldn't happen, but see comment in process_gpg_verify_result() */ - if (g_hash_table_contains (pull_data->verified_commits, checksum)) - return TRUE; + /* Shouldn't happen, but see comment in process_gpg_verify_result() */ + if ((!pull_data->gpg_verify || g_hash_table_contains (pull_data->verified_commits, checksum)) + && (!pull_data->signapi_commit_verifiers || g_hash_table_contains (pull_data->signapi_verified_commits, checksum))) + return TRUE; g_autoptr(GBytes) signed_data = g_variant_get_data_as_bytes (commit); @@ -382,12 +388,13 @@ _verify_unwritten_commit (OtPullData *pull_data, if (detached_metadata == NULL) return glnx_throw (error, "Can't verify commit without detached metadata"); - if (!_sign_verify_for_remote (pull_data->signapi_commit_verifiers, signed_data, detached_metadata, error)) + g_autofree char *success_message = NULL; + if (!_sign_verify_for_remote (pull_data->signapi_commit_verifiers, signed_data, detached_metadata, &success_message, error)) return glnx_prefix_error (error, "Can't verify commit"); /* Mark the commit as verified to avoid double verification * see process_verify_result () for rationale */ - g_hash_table_add (pull_data->verified_commits, g_strdup (checksum)); + g_hash_table_insert (pull_data->signapi_verified_commits, g_strdup (checksum), g_steal_pointer (&success_message)); } return TRUE; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index fbcfc8a6..5a276e62 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1541,11 +1541,12 @@ scan_commit_object (OtPullData *pull_data, #endif /* OSTREE_DISABLE_GPGME */ if (pull_data->signapi_commit_verifiers && - !g_hash_table_contains (pull_data->verified_commits, checksum)) + !g_hash_table_contains (pull_data->signapi_verified_commits, checksum)) { g_autoptr(GError) last_verification_error = NULL; gboolean found_any_signature = FALSE; gboolean found_valid_signature = FALSE; + g_autofree char *success_message = NULL; for (guint i = 0; i < pull_data->signapi_commit_verifiers->len; i++) { @@ -1557,6 +1558,7 @@ scan_commit_object (OtPullData *pull_data, if (ostree_sign_commit_verify (sign, pull_data->repo, checksum, + &success_message, cancellable, last_verification_error ? NULL : &last_verification_error)) { @@ -1574,6 +1576,8 @@ scan_commit_object (OtPullData *pull_data, g_propagate_error (error, g_steal_pointer (&last_verification_error)); return glnx_prefix_error (error, "Can't verify commit %s", checksum); } + g_assert (success_message); + g_hash_table_insert (pull_data->signapi_verified_commits, g_strdup (checksum), g_steal_pointer (&success_message)); } /* If we found a legacy transaction flag, assume we have to scan. @@ -3469,6 +3473,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, (GDestroyNotify)g_free); pull_data->verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal, (GDestroyNotify)g_free, NULL); + pull_data->signapi_verified_commits = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify)g_free, NULL); pull_data->ref_keyring_map = g_hash_table_new_full (ostree_collection_ref_hash, ostree_collection_ref_equal, (GDestroyNotify)ostree_collection_ref_free, (GDestroyNotify)g_free); pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, @@ -3962,7 +3968,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_assert (pull_data->signapi_summary_verifiers); - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, &temp_error)) + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error)) { if (summary_from_cache) { @@ -3991,7 +3997,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, cancellable, error)) goto out; - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, error)) + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error)) goto out; } else @@ -4546,6 +4552,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, const guint n_seconds = (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC); g_autofree char *formatted_xferred = g_format_size (bytes_transferred); g_string_append_printf (msg, "\ntransfer: secs: %u size: %s", n_seconds, formatted_xferred); + if (pull_data->signapi_commit_verifiers) + { + g_assert_cmpuint (g_hash_table_size (pull_data->signapi_verified_commits), >, 0); + } ot_journal_send ("MESSAGE=%s", msg->str, "MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_MESSAGE_FETCH_COMPLETE_ID), @@ -4622,6 +4632,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->ref_original_commits, (GDestroyNotify) g_hash_table_unref); g_free (pull_data->timestamp_check_from_rev); g_clear_pointer (&pull_data->verified_commits, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->signapi_verified_commits, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->ref_keyring_map, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref); @@ -6114,7 +6125,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, sig_variant = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, signatures, FALSE); - if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, error)) + if (!_sign_verify_for_remote (signapi_summary_verifiers, summary, sig_variant, NULL, error)) goto out; } } diff --git a/src/libostree/ostree-sign-dummy.c b/src/libostree/ostree-sign-dummy.c index 82575dc5..56f10d6e 100644 --- a/src/libostree/ostree-sign-dummy.c +++ b/src/libostree/ostree-sign-dummy.c @@ -154,6 +154,7 @@ const gchar * ostree_sign_dummy_metadata_format (OstreeSign *self) gboolean ostree_sign_dummy_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error) { if (!check_dummy_sign_enabled (error)) @@ -182,7 +183,11 @@ gboolean ostree_sign_dummy_data_verify (OstreeSign *self, g_debug("Stored signature %d: %s", (gint)i, sign->pk_ascii); if (!g_strcmp0(sign_ascii, sign->pk_ascii)) - return TRUE; + { + if (out_success_message) + *out_success_message = g_strdup ("dummy: Signature verified"); + return TRUE; + } else return glnx_throw (error, "signature: dummy: incorrect signature %" G_GSIZE_FORMAT, i); } diff --git a/src/libostree/ostree-sign-dummy.h b/src/libostree/ostree-sign-dummy.h index c37bcdfa..bf5d63a1 100644 --- a/src/libostree/ostree-sign-dummy.h +++ b/src/libostree/ostree-sign-dummy.h @@ -63,6 +63,7 @@ gboolean ostree_sign_dummy_data (OstreeSign *self, gboolean ostree_sign_dummy_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **success_message, GError **error); const gchar * ostree_sign_dummy_metadata_key (OstreeSign *self); diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 05fbe5eb..ed2bf977 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -169,6 +169,7 @@ _compare_ed25519_keys(gconstpointer a, gconstpointer b) { gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error) { g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); @@ -243,8 +244,12 @@ gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, } else { - g_debug ("Signature verified successfully with key '%s'", - sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + if (out_success_message) + { + *out_success_message = + g_strdup_printf ("ed25519: Signature verified successfully with key '%s'", + sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, public_key->data, crypto_sign_PUBLICKEYBYTES)); + } return TRUE; } } diff --git a/src/libostree/ostree-sign-ed25519.h b/src/libostree/ostree-sign-ed25519.h index 76c7e14d..72152eab 100644 --- a/src/libostree/ostree-sign-ed25519.h +++ b/src/libostree/ostree-sign-ed25519.h @@ -61,6 +61,7 @@ gboolean ostree_sign_ed25519_data (OstreeSign *self, gboolean ostree_sign_ed25519_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error); const gchar * ostree_sign_ed25519_get_name (OstreeSign *self); diff --git a/src/libostree/ostree-sign.c b/src/libostree/ostree-sign.c index f3992480..bcb5d0a6 100644 --- a/src/libostree/ostree-sign.c +++ b/src/libostree/ostree-sign.c @@ -322,13 +322,14 @@ gboolean ostree_sign_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error) { g_return_val_if_fail (OSTREE_IS_SIGN (self), FALSE); if (OSTREE_SIGN_GET_IFACE (self)->data_verify == NULL) return glnx_throw (error, "not implemented"); - return OSTREE_SIGN_GET_IFACE (self)->data_verify(self, data, signatures, error); + return OSTREE_SIGN_GET_IFACE (self)->data_verify(self, data, signatures, out_success_message, error); } /* @@ -389,6 +390,7 @@ gboolean ostree_sign_commit_verify (OstreeSign *self, OstreeRepo *repo, const gchar *commit_checksum, + char **out_success_message, GCancellable *cancellable, GError **error) @@ -427,6 +429,7 @@ ostree_sign_commit_verify (OstreeSign *self, return ostree_sign_data_verify (self, signed_data, signatures, + out_success_message, error); } diff --git a/src/libostree/ostree-sign.h b/src/libostree/ostree-sign.h index 588ace53..0d069059 100644 --- a/src/libostree/ostree-sign.h +++ b/src/libostree/ostree-sign.h @@ -72,6 +72,7 @@ struct _OstreeSignInterface gboolean (* data_verify) (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error); const gchar *(* metadata_key) (OstreeSign *self); const gchar *(* metadata_format) (OstreeSign *self); @@ -105,6 +106,7 @@ _OSTREE_PUBLIC gboolean ostree_sign_data_verify (OstreeSign *self, GBytes *data, GVariant *signatures, + char **out_success_message, GError **error); _OSTREE_PUBLIC @@ -124,6 +126,7 @@ _OSTREE_PUBLIC gboolean ostree_sign_commit_verify (OstreeSign *self, OstreeRepo *repo, const gchar *commit_checksum, + char **out_success_message, GCancellable *cancellable, GError **error); diff --git a/src/ostree/ot-builtin-sign.c b/src/ostree/ot-builtin-sign.c index d6cc167a..c7777489 100644 --- a/src/ostree/ot-builtin-sign.c +++ b/src/ostree/ot-builtin-sign.c @@ -70,6 +70,7 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, g_autoptr (OstreeRepo) repo = NULL; g_autoptr (OstreeSign) sign = NULL; g_autofree char *resolved_commit = NULL; + g_autofree char *success_message = NULL; const char *commit; char **key_ids; int n_key_ids, ii; @@ -130,9 +131,12 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, if (ostree_sign_commit_verify (sign, repo, resolved_commit, + &success_message, cancellable, &local_error)) { + g_assert (success_message); + g_print ("%s\n", success_message); ret = TRUE; goto out; } @@ -180,9 +184,13 @@ ostree_builtin_sign (int argc, char **argv, OstreeCommandInvocation *invocation, if (ostree_sign_commit_verify (sign, repo, resolved_commit, + &success_message, cancellable, error)) - ret = TRUE; + { + g_print ("%s\n", success_message); + ret = TRUE; + } } /* Check via file */ } else diff --git a/tests/test-signed-commit.sh b/tests/test-signed-commit.sh index 6bdbfdd6..d43efef7 100755 --- a/tests/test-signed-commit.sh +++ b/tests/test-signed-commit.sh @@ -121,8 +121,10 @@ done ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy ${COMMIT} ${DUMMYSIGN} ${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=ed25519 ${COMMIT} ${SECRET} # and verify -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 ${COMMIT} ${PUBLIC} >out.txt +assert_file_has_content out.txt "ed25519: Signature verified successfully with key" +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --sign-type=dummy --verify ${COMMIT} ${DUMMYSIGN} >out.txt +assert_file_has_content out.txt "dummy: Signature verified" echo "ok multiple signing " # Prepare files with public ed25519 signatures @@ -140,7 +142,8 @@ fi # Test with single key in list echo ${PUBLIC} > ${PUBKEYS} -${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} +${CMD_PREFIX} ostree --repo=${test_tmpdir}/repo sign --verify --sign-type=ed25519 --keys-file=${PUBKEYS} ${COMMIT} >out.txt +assert_file_has_content out.txt 'ed25519: Signature verified successfully' # Test the file with multiple keys without a valid public key for((i=0;i<100;i++)); do From eb3fe35b0684ee6e2c4c514a49e8151c00ba59a0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 17 Jun 2020 10:38:06 -0400 Subject: [PATCH 161/177] ci: Import latest ci-commitmessage-submodules from rpm-ostree Especially for https://github.com/coreos/rpm-ostree/pull/2079. --- ci/ci-commitmessage-submodules.sh | 35 +++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/ci/ci-commitmessage-submodules.sh b/ci/ci-commitmessage-submodules.sh index 35d828e4..edc0d780 100755 --- a/ci/ci-commitmessage-submodules.sh +++ b/ci/ci-commitmessage-submodules.sh @@ -1,5 +1,8 @@ #!/bin/bash -set -euo pipefail +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libbuild.sh # Copyright 2017 Colin Walters # Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php) @@ -13,11 +16,8 @@ set -euo pipefail # It's very common for people to accidentally change submodules, and having this # requirement is a small hurdle to pass. -# if running under PAPR, use the branch/PR HEAD actually -# being tested rather than the merge sha -HEAD=${PAPR_COMMIT:-HEAD} -dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +# If passed the commit, use that. Otherwise, just use HEAD. +HEAD=${1:-HEAD} tmpd=$(mktemp -d) touch ${tmpd}/.tmpdir @@ -30,8 +30,8 @@ cleanup_tmp() { trap cleanup_tmp EXIT if ! [ -x /usr/bin/git ]; then - pkg_upgrade - pkg_install git + pkg_upgrade + pkg_install git fi gitdir=$(realpath $(pwd)) @@ -42,17 +42,34 @@ cp -a ${gitdir} ${tmpd}/workdir cd ${tmpd}/workdir git log --pretty=oneline origin/master..$HEAD | while read logline; do commit=$(echo ${logline} | cut -f 1 -d ' ') + # For merge commits, just check that they're empty (i.e. no conflict + # resolution was needed). Otherwise, let's just error out. Conflicts should + # be resolved by rebasing the PR. + # https://stackoverflow.com/questions/3824050#comment82244548_13956422 + if [ "$(git rev-list --no-walk --count --merges ${commit})" -ne 0 ]; then + if [ -n "$(git diff-tree ${commit})" ]; then + echo "error: non-empty git merge: resolve conflicts by rebasing!" + exit 1 + fi + echo "Commit ${commit} is an empty merge commit; ignoring..." + continue + fi git diff --name-only ${commit}^..${commit} > ${tmpd}/diff.txt git log -1 ${commit} > ${tmpd}/log.txt echo "Validating commit for submodules: $commit" + sed -e 's,^,# ,' < ${tmpd}/log.txt git checkout -q "${commit}" git submodule update --init git submodule foreach --quiet 'echo $path'| while read submodule; do if grep -q -e '^'${submodule} ${tmpd}/diff.txt; then echo "Commit $commit modifies submodule: $submodule" expected_match="Update submodule: $submodule" + # check if it's from dependabot + if grep -q -e '^Author: dependabot' ${tmpd}/log.txt; then + echo "Commit $commit contains bump from Dependabot" + continue + fi if ! grep -q -e "$expected_match" ${tmpd}/log.txt; then - sed -e 's,^,# ,' < ${tmpd}/log.txt echo "error: Commit message for ${commit} changes a submodule, but does not match regex ${expected_match}" exit 1 fi From ce73876389a062dbf169e4231f507200b05ee41b Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 17 Jun 2020 16:28:32 +0300 Subject: [PATCH 162/177] sign/ed25519: fix the abort in case of incorrect public key We need to check the size of public key before trying to use it. Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 05fbe5eb..0aaad189 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -415,12 +415,12 @@ gboolean ostree_sign_ed25519_add_pk (OstreeSign *self, return glnx_throw (error, "Unknown ed25519 public key type"); } - g_autofree char *hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); - g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); - if (n_elements != crypto_sign_PUBLICKEYBYTES) return glnx_throw (error, "Incorrect ed25519 public key"); + g_autofree char *hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_debug ("Read ed25519 public key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); + if (g_list_find_custom (sign->public_keys, key, _compare_ed25519_keys) == NULL) { gpointer newkey = g_memdup (key, n_elements); @@ -449,14 +449,14 @@ _ed25519_add_revoked (OstreeSign *self, gsize n_elements = 0; gpointer key = g_base64_decode (rk_ascii, &n_elements); - g_autofree char * hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); - g_debug ("Read ed25519 revoked key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); - if (n_elements != crypto_sign_PUBLICKEYBYTES) { return glnx_throw (error, "Incorrect ed25519 revoked key"); } + g_autofree char * hex = g_malloc0 (crypto_sign_PUBLICKEYBYTES*2 + 1); + g_debug ("Read ed25519 revoked key = %s", sodium_bin2hex (hex, crypto_sign_PUBLICKEYBYTES*2+1, key, n_elements)); + if (g_list_find_custom (sign->revoked_keys, key, _compare_ed25519_keys) == NULL) { gpointer newkey = g_memdup (key, n_elements); From ede93dc2ef3e66caf28e54d3394784877d3bbe6f Mon Sep 17 00:00:00 2001 From: Denis Pynkin Date: Wed, 17 Jun 2020 16:21:35 +0000 Subject: [PATCH 163/177] sign/ed25519: fix return value if no correct keys in file Fix the return value if file doesn't contains correct public key(s). Signed-off-by: Denis Pynkin --- src/libostree/ostree-sign-ed25519.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libostree/ostree-sign-ed25519.c b/src/libostree/ostree-sign-ed25519.c index 0aaad189..ed6d0c23 100644 --- a/src/libostree/ostree-sign-ed25519.c +++ b/src/libostree/ostree-sign-ed25519.c @@ -546,6 +546,8 @@ _load_pk_from_file (OstreeSign *self, return glnx_throw (error, "signature: ed25519: no valid keys in file '%s'", filename); + else + return FALSE; } return TRUE; From 933683719471034ff0a5828b4fa66a29fad8321e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 16 Jun 2020 13:02:53 +0000 Subject: [PATCH 164/177] libostree-devel.sym: Remove nonexistent stub symbol This should have been removed when we added symbols to this list. --- src/libostree/libostree-devel.sym | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index fd0ffd57..5999aabe 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -21,7 +21,6 @@ LIBOSTREE_2020.2 { global: ostree_repo_commit_modifier_set_sepolicy_from_commit; - someostree_symbol_deleteme; ostree_sign_get_type; ostree_sign_get_all; ostree_sign_commit; From 516c1340b3f8c352f487abc67c9d1f4ab8fac975 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 17 Jun 2020 15:48:31 -0400 Subject: [PATCH 165/177] ci: Remove libpaprci/ directory And move everything that was in it directly in `ci/`. There's a bunch more cleanups here that we need to do (and more changes to upstream from the rpm-ostree copies of this). --- ci/{libpaprci => }/Makefile.dist-packaging | 0 ci/build-check.sh | 2 +- ci/build-rpm.sh | 7 +++---- ci/build.sh | 2 +- ci/flatpak.sh | 2 +- ci/installdeps.sh | 2 +- ci/{libpaprci => }/libbuild.sh | 0 ci/{libpaprci => }/make-git-snapshot.sh | 0 ci/provision-prep.sh | 2 +- ci/{libpaprci => }/rpmbuild-cwd | 0 ci/rpmostree.sh | 2 +- 11 files changed, 9 insertions(+), 10 deletions(-) rename ci/{libpaprci => }/Makefile.dist-packaging (100%) rename ci/{libpaprci => }/libbuild.sh (100%) rename ci/{libpaprci => }/make-git-snapshot.sh (100%) rename ci/{libpaprci => }/rpmbuild-cwd (100%) diff --git a/ci/libpaprci/Makefile.dist-packaging b/ci/Makefile.dist-packaging similarity index 100% rename from ci/libpaprci/Makefile.dist-packaging rename to ci/Makefile.dist-packaging diff --git a/ci/build-check.sh b/ci/build-check.sh index af276ef0..aed8ba7e 100755 --- a/ci/build-check.sh +++ b/ci/build-check.sh @@ -4,7 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh ${dn}/build.sh topdir=$(git rev-parse --show-toplevel) resultsdir=$(mktemp -d) diff --git a/ci/build-rpm.sh b/ci/build-rpm.sh index 5d096333..1b67285e 100755 --- a/ci/build-rpm.sh +++ b/ci/build-rpm.sh @@ -4,8 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) -paprcidir=${dn}/libpaprci -. ${paprcidir}/libbuild.sh +. ${dn}/libbuild.sh # Auto-provision bootstrap resources if run as root (normally in CI) if test "$(id -u)" == 0; then @@ -34,13 +33,13 @@ case "${CONFIGOPTS:-}" in esac # TODO: Use some form of rpm's --build-in-place to skip archive-then-unpack? -make -f ${paprcidir}/Makefile.dist-packaging srpm PACKAGE=libostree DISTGIT_NAME=ostree +make -f ${dn}/Makefile.dist-packaging srpm PACKAGE=libostree DISTGIT_NAME=ostree if test "$(id -u)" == 0; then pkg_builddep *.src.rpm else echo "NOTE: Running as non-root, assuming build dependencies are installed" fi -if ! ${paprcidir}/rpmbuild-cwd --rebuild *.src.rpm; then +if ! ${dn}/rpmbuild-cwd --rebuild *.src.rpm; then find . -type f -name config.log -exec cat {} \; exit 1 fi diff --git a/ci/build.sh b/ci/build.sh index 09015074..4ff6eaa5 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -4,7 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh ${dn}/installdeps.sh diff --git a/ci/flatpak.sh b/ci/flatpak.sh index 989b1235..e155d897 100755 --- a/ci/flatpak.sh +++ b/ci/flatpak.sh @@ -9,7 +9,7 @@ set -xeuo pipefail FLATPAK_TAG=1.4.1 dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh codedir=$(pwd) diff --git a/ci/installdeps.sh b/ci/installdeps.sh index 4fc3280f..7d7c723e 100755 --- a/ci/installdeps.sh +++ b/ci/installdeps.sh @@ -12,7 +12,7 @@ if [ -n "${SKIP_INSTALLDEPS:-}" ]; then fi dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh pkg_upgrade pkg_install_buildroot diff --git a/ci/libpaprci/libbuild.sh b/ci/libbuild.sh similarity index 100% rename from ci/libpaprci/libbuild.sh rename to ci/libbuild.sh diff --git a/ci/libpaprci/make-git-snapshot.sh b/ci/make-git-snapshot.sh similarity index 100% rename from ci/libpaprci/make-git-snapshot.sh rename to ci/make-git-snapshot.sh diff --git a/ci/provision-prep.sh b/ci/provision-prep.sh index 30825802..96a2041e 100755 --- a/ci/provision-prep.sh +++ b/ci/provision-prep.sh @@ -4,7 +4,7 @@ set -xeuo pipefail dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh pkg_upgrade pkg_install_buildroot pkg_install sudo which attr fuse strace \ diff --git a/ci/libpaprci/rpmbuild-cwd b/ci/rpmbuild-cwd similarity index 100% rename from ci/libpaprci/rpmbuild-cwd rename to ci/rpmbuild-cwd diff --git a/ci/rpmostree.sh b/ci/rpmostree.sh index 27fefaf6..0a5f5df9 100755 --- a/ci/rpmostree.sh +++ b/ci/rpmostree.sh @@ -9,7 +9,7 @@ set -xeuo pipefail RPMOSTREE_TAG=v2019.4 dn=$(dirname $0) -. ${dn}/libpaprci/libbuild.sh +. ${dn}/libbuild.sh codedir=$(pwd) From d21181653ecb678dfa2a6ace422ece17d4e2de77 Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Wed, 17 Jun 2020 14:35:18 -0700 Subject: [PATCH 166/177] tests: Check that example symbol isn't released For the motivation for this see #2132. --- tests/kolainst/libtest-core.sh | 10 ++++++++++ tests/test-symbols.sh | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/kolainst/libtest-core.sh b/tests/kolainst/libtest-core.sh index 945d2857..64b3e0a3 100644 --- a/tests/kolainst/libtest-core.sh +++ b/tests/kolainst/libtest-core.sh @@ -115,6 +115,16 @@ assert_file_has_content () { done } +assert_file_has_content_once () { + fpath=$1 + shift + for re in "$@"; do + if ! test $(grep -e "$re" "$fpath" | wc -l) = "1"; then + _fatal_print_file "$fpath" "File '$fpath' doesn't match regexp '$re' exactly once" + fi + done +} + assert_file_has_content_literal () { fpath=$1; shift for s in "$@"; do diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index f6742d93..8f67a46c 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -21,6 +21,8 @@ set -xeuo pipefail +. $(dirname $0)/libtest.sh + echo '1..3' released_syms=${G_TEST_SRCDIR}/src/libostree/libostree-released.sym @@ -41,6 +43,16 @@ echo "Verifying all expected symbols are actually exported..." grep --no-filename ' ostree_[A-Za-z0-9_]*;' ${released_syms} ${devel_syms} ${experimental_sym} | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9A-Z_.]*,\1,' |sort -u > found-symbols.txt diff -u expected-symbols.txt found-symbols.txt + +echo "Checking that the example symbol wasn't copy-pasted..." +if test -f ${devel_syms}; then + assert_file_has_content_once ${devel_syms} "someostree_symbol_deleteme" +fi +if test -f ${experimental_sym}; then + assert_not_file_has_content ${experimental_sym} "someostree_symbol_deleteme" +fi +assert_not_file_has_content ${released_syms} "someostree_symbol_deleteme" + echo "ok exports" # cmd__private__ is private. The fetcher symbol should not have been made public. From c87a4c71bc72f91ecd49320d3bf78aef08d97033 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Jun 2020 20:57:53 +0000 Subject: [PATCH 167/177] core: Add documentation for ostree_commit_get_timestamp() Working on some rpm-ostree bits and was going to pass this to the `chrono` crate and I forgot the format, went to look at the docs. Oops. --- src/libostree/ostree-core.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 48815bd3..523f57c0 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -2376,6 +2376,13 @@ ostree_commit_get_parent (GVariant *commit_variant) return ostree_checksum_from_bytes_v (bytes); } +/** + * ostree_commit_get_timestamp: + * @commit_variant: Commit object + * + * Returns: timestamp in seconds since the Unix epoch, UTC + * Since: 2016.3 + */ guint64 ostree_commit_get_timestamp (GVariant *commit_variant) { From 935f879542c6d3a48903ac1cd9d90e590845b582 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 18 Jun 2020 21:19:43 +0000 Subject: [PATCH 168/177] sysroot: Remove unimplemented ostree_sysroot_lock_with_mount_namespace This came in with 5af403be0cc64df50ad21cef05f3268ead256d6d but was never implemented. I noticed this now because the Rust ostree bindings generate a wrapper for it which the linker tries to use. --- src/libostree/ostree-sysroot.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libostree/ostree-sysroot.h b/src/libostree/ostree-sysroot.h index af8192fc..d9f5a546 100644 --- a/src/libostree/ostree-sysroot.h +++ b/src/libostree/ostree-sysroot.h @@ -101,9 +101,6 @@ GFile * ostree_sysroot_get_deployment_origin_path (GFile *deployment_path); _OSTREE_PUBLIC gboolean ostree_sysroot_lock (OstreeSysroot *self, GError **error); -_OSTREE_PUBLIC -gboolean ostree_sysroot_lock_with_mount_namespace (OstreeSysroot *self, GError **error); - _OSTREE_PUBLIC gboolean ostree_sysroot_try_lock (OstreeSysroot *self, gboolean *out_acquired, From 5aa22e0b1fee3c517d1df2d44c091f168fece01c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 19 Jun 2020 12:55:33 +0000 Subject: [PATCH 169/177] tests: Port to Debian autopkgtest reboot API See https://github.com/coreos/coreos-assembler/pull/1528 I think we can drop the old cosa reboot APIs after this, though I've already forgotten where else I might have written tests using it. --- tests/kolainst/destructive/staged-delay.sh | 10 +++++----- tests/kolainst/destructive/staged-deploy.sh | 11 +++++------ tests/kolainst/destructive/var-mount.sh | 11 +++++------ tests/kolainst/libinsttest.sh | 7 ------- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/kolainst/destructive/staged-delay.sh b/tests/kolainst/destructive/staged-delay.sh index a320202f..5671bbea 100755 --- a/tests/kolainst/destructive/staged-delay.sh +++ b/tests/kolainst/destructive/staged-delay.sh @@ -9,9 +9,8 @@ set -xeuo pipefail require_writable_sysroot prepare_tmpdir -n=$(nth_boot) -case "${n}" in -1) +case "${AUTOPKGTEST_REBOOT_MARK:-}" in +"") dropin=/etc/systemd/system/ostree-finalize-staged.service.d/delay.conf mkdir -p $(dirname ${dropin}) cat >"${dropin}" << 'EOF' @@ -21,10 +20,10 @@ ExecStop=/bin/sh -c 'sleep 10 && if ! test -d /boot/loader/entries; then echo er EOF systemctl daemon-reload rpm-ostree kargs --append=somedummykarg=1 -kola_reboot +/tmp/autopkgtest-reboot 2 ;; -2) +"2") journalctl -b -1 -u ostree-finalize-staged > logs.txt assert_file_has_content_literal logs.txt 'ostree-finalize-staged found /boot/loader/entries' # older systemd doesn't output the success message @@ -36,5 +35,6 @@ else fi assert_file_has_content_literal /proc/cmdline somedummykarg=1 ;; +*) fatal "Unexpected AUTOPKGTEST_REBOOT_MARK=${AUTOPKGTEST_REBOOT_MARK}" ;; esac echo ok diff --git a/tests/kolainst/destructive/staged-deploy.sh b/tests/kolainst/destructive/staged-deploy.sh index cc3e7a7e..a5da0119 100755 --- a/tests/kolainst/destructive/staged-deploy.sh +++ b/tests/kolainst/destructive/staged-deploy.sh @@ -6,9 +6,8 @@ set -xeuo pipefail require_writable_sysroot prepare_tmpdir -n=$(nth_boot) -case "${n}" in - 1) +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") # Initial cleanup to handle the cosa fast-build case ## TODO remove workaround for https://github.com/coreos/rpm-ostree/pull/2021 mkdir -p /var/lib/rpm-ostree/history @@ -48,9 +47,9 @@ case "${n}" in fatal "Pinned staged deployment" fi assert_file_has_content err.txt 'Cannot pin staged deployment' - kola_reboot + /tmp/autopkgtest-reboot "2" ;; - 2) + "2") # Check that deploy-staged service worked rpm-ostree status # Assert that the previous boot had a journal entry for it @@ -118,5 +117,5 @@ case "${n}" in ostree refs --delete staged-deploy nonstaged-deploy echo "ok cleanup refs" ;; - *) fatal "Unexpected boot count" ;; + *) fatal "Unexpected AUTOPKGTEST_REBOOT_MARK=${AUTOPKGTEST_REBOOT_MARK}" ;; esac diff --git a/tests/kolainst/destructive/var-mount.sh b/tests/kolainst/destructive/var-mount.sh index b3fe3d98..86c69fd1 100755 --- a/tests/kolainst/destructive/var-mount.sh +++ b/tests/kolainst/destructive/var-mount.sh @@ -4,20 +4,19 @@ set -xeuo pipefail . ${KOLA_EXT_DATA}/libinsttest.sh -n=$(nth_boot) -case "${n}" in - 1) +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") require_writable_sysroot # Hack this off for now chattr -i /sysroot cp -a /var /sysroot/myvar touch /sysroot/myvar/somenewfile echo '/sysroot/myvar /var none bind 0 0' >> /etc/fstab - kola_reboot + /tmp/autopkgtest-reboot "2" ;; - 2) + "2") systemctl status var.mount test -f /var/somenewfile ;; - *) fatal "Unexpected boot count $n" + *) fatal "Unexpected AUTOPKGTEST_REBOOT_MARK=${AUTOPKGTEST_REBOOT_MARK}" ;; esac diff --git a/tests/kolainst/libinsttest.sh b/tests/kolainst/libinsttest.sh index 2552cf78..bf40a285 100644 --- a/tests/kolainst/libinsttest.sh +++ b/tests/kolainst/libinsttest.sh @@ -66,13 +66,6 @@ nth_boot() { journalctl --list-boots | wc -l } -kola_reboot() { - kill -TERM $$ - sleep 2m - echo "failed to reboot?" 1>&2 - exit 1 -} - # Determine our origin refspec - we'll use this as a test base rpmostree=$(which rpm-ostree 2>/dev/null) if test -z "${rpmostree}"; then From 1d755f62af8bcd2c094324ddd3d591680d41726c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 30 Jun 2020 15:05:19 -0400 Subject: [PATCH 170/177] lib/repo: Handle EACCES for POSIX locking If `glnx_make_lock_file` falls back to `flock`, on NFS this uses POSIX locks (`F_SETLK`). As such, we need to be able to handle `EACCES` as well as `EAGAIN` (see `fnctl(2)`). I think this is what coreos-ostree-importer has been hitting, which runs on RHEL7 in the Fedora infra and does locking over an NFS share where multiple apps could concurrently pull things into the repo. --- src/libostree/ostree-repo-commit.c | 2 +- src/libostree/ostree-repo.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 7dd68e96..727e853b 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1933,7 +1933,7 @@ rename_pending_loose_objects (OstreeRepo *self, return TRUE; } -/* Try to lock a transaction stage directory created by +/* Try to lock and delete a transaction stage directory created by * ostree_repo_prepare_transaction(). */ static gboolean diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 5fd92bb9..82dd286a 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5980,7 +5980,9 @@ _ostree_repo_try_lock_tmpdir (int tmpdir_dfd, if (!glnx_make_lock_file (tmpdir_dfd, lock_name, LOCK_EX | LOCK_NB, file_lock_out, &local_error)) { - if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK)) + /* we need to handle EACCES too in the case of POSIX locks; see F_SETLK in fcntl(2) */ + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_WOULD_BLOCK) + || g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) { did_lock = FALSE; } From 892e9acf2593ce7b99b7e4616b4a6705ca37ae6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Danis?= Date: Thu, 16 Apr 2020 17:17:12 +0200 Subject: [PATCH 171/177] lib/deltas: convert ostree_repo_static_delta_generate to new style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "new style" code generally avoids `goto err` because it conflicts with `__attribute__((cleanup))`. This fixes a compiler warning. Signed-off-by: Frédéric Danis --- .../ostree-repo-static-delta-compilation.c | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 88e9ddf6..3e9a8e30 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -1346,7 +1346,6 @@ ostree_repo_static_delta_generate (OstreeRepo *self, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; OstreeStaticDeltaBuilder builder = { 0, }; guint i; guint min_fallback_size; @@ -1365,8 +1364,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) detached = NULL; gboolean inline_parts; guint endianness = G_BYTE_ORDER; - builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref); - builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); + g_autoptr(GPtrArray) builder_parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref); + g_autoptr(GPtrArray) builder_fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); g_auto(GLnxTmpfile) descriptor_tmpf = { 0, }; g_autoptr(OtVariantBuilder) descriptor_builder = NULL; @@ -1385,6 +1384,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE); builder.swap_endian = endianness != G_BYTE_ORDER; + builder.parts = builder_parts; + builder.fallback_objects = builder_fallback_objects; { gboolean use_bsdiff; if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff)) @@ -1408,7 +1409,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, to, &to_commit, error)) - goto out; + return FALSE; builder.delta_opts = delta_opts; @@ -1419,7 +1420,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, descriptor_name = g_strdup (glnx_basename (opt_filename)); if (!glnx_opendirat (AT_FDCWD, dn, TRUE, &descriptor_dfd, error)) - goto out; + return FALSE; } else { @@ -1428,9 +1429,9 @@ ostree_repo_static_delta_generate (OstreeRepo *self, const char *dn = dirname (dnbuf); if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, dn, DEFAULT_DIRECTORY_MODE, cancellable, error)) - goto out; + return FALSE; if (!glnx_opendirat (self->repo_dir_fd, dn, TRUE, &descriptor_dfd, error)) - goto out; + return FALSE; descriptor_name = g_strdup (basename (descriptor_relpath)); } @@ -1439,17 +1440,17 @@ ostree_repo_static_delta_generate (OstreeRepo *self, /* Ignore optimization flags */ if (!generate_delta_lowlatency (self, from, to, delta_opts, &builder, cancellable, error)) - goto out; + return FALSE; if (!glnx_open_tmpfile_linkable_at (descriptor_dfd, ".", O_WRONLY | O_CLOEXEC, &descriptor_tmpf, error)) - goto out; + return FALSE; descriptor_builder = ot_variant_builder_new (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), descriptor_tmpf.fd); /* Open the metadata dict */ if (!ot_variant_builder_open (descriptor_builder, G_VARIANT_TYPE ("a{sv}"), error)) - goto out; + return FALSE; /* NOTE: Add user-supplied metadata first. This is used by at least * flatpak as a way to provide MIME content sniffing, since the @@ -1464,7 +1465,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, while ((item = g_variant_iter_next_value (&iter))) { if (!ot_variant_builder_add_value (descriptor_builder, item, error)) - goto out; + return FALSE; g_variant_unref (item); } } @@ -1483,7 +1484,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_assert_not_reached (); } if (!ot_variant_builder_add (descriptor_builder, error, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char))) - goto out; + return FALSE; } part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT)); @@ -1504,21 +1505,18 @@ ostree_repo_static_delta_generate (OstreeRepo *self, !ot_variant_builder_add_from_fd (descriptor_builder, G_VARIANT_TYPE ("(yay)"), part_builder->part_tmpf.fd, part_builder->compressed_size, error) || !ot_variant_builder_close (descriptor_builder, error) || !ot_variant_builder_close (descriptor_builder, error)) - goto out; + return FALSE; } else { g_autofree char *partstr = g_strdup_printf ("%u", i); if (fchmod (part_builder->part_tmpf.fd, 0644) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fchmod"); if (!glnx_link_tmpfile_at (&part_builder->part_tmpf, GLNX_LINK_TMPFILE_REPLACE, descriptor_dfd, partstr, error)) - goto out; + return FALSE; } g_variant_builder_add_value (part_headers, part_builder->header); @@ -1529,21 +1527,21 @@ ostree_repo_static_delta_generate (OstreeRepo *self, if (!get_fallback_headers (self, &builder, &fallback_headers, cancellable, error)) - goto out; + return FALSE; if (!ostree_repo_read_commit_detached_metadata (self, to, &detached, cancellable, error)) - goto out; + return FALSE; if (detached) { g_autofree char *detached_key = _ostree_get_relative_static_delta_path (from, to, "commitmeta"); if (!ot_variant_builder_add (descriptor_builder, error, "{sv}", detached_key, detached)) - goto out; + return FALSE; } /* Close metadata dict */ if (!ot_variant_builder_close (descriptor_builder, error)) - goto out; + return FALSE; /* Generate OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ { @@ -1568,10 +1566,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_variant_builder_end (part_headers), error) || !ot_variant_builder_add_value (descriptor_builder, fallback_headers, error)) - goto out; + return FALSE; if (!ot_variant_builder_end (descriptor_builder, error)) - goto out; + return FALSE; g_date_time_unref (now); } @@ -1589,18 +1587,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } if (fchmod (descriptor_tmpf.fd, 0644) < 0) - { - glnx_set_error_from_errno (error); - goto out; - } + return glnx_throw_errno_prefix (error, "fchmod"); if (!glnx_link_tmpfile_at (&descriptor_tmpf, GLNX_LINK_TMPFILE_REPLACE, descriptor_dfd, descriptor_name, error)) - goto out; + return FALSE; - ret = TRUE; - out: - g_clear_pointer (&builder.parts, g_ptr_array_unref); - g_clear_pointer (&builder.fallback_objects, g_ptr_array_unref); - return ret; + return TRUE; } From f924b0bbe2af1dbeb9abbd6a40c216c08a306739 Mon Sep 17 00:00:00 2001 From: William Manley Date: Wed, 8 Jul 2020 15:54:22 +0100 Subject: [PATCH 172/177] OWNERS: Uncomment @wmanley I've made my affiliation public now thanks to @cgwalters: https://github.com/ostreedev/ostree/pull/1678#issuecomment-653148139 --- OWNERS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/OWNERS b/OWNERS index 47a7bc81..4e8de5f5 100644 --- a/OWNERS +++ b/OWNERS @@ -10,8 +10,7 @@ approvers: - cgwalters - jlebon - pwithnall - # Does not currently have affiliation public - # - wmanley + - wmanley reviewers: - lucab - mwleeds From 92284f9b8107078d93423f772929bed0ab15e75b Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 16 Jul 2020 11:42:57 -0400 Subject: [PATCH 173/177] ci: Constrain parallel build jobs The default `_NPROCESSORS_ONLN` heuristic we have isn't cgroups aware. So it thinks it has e.g. 40 CPUs when running in a k8s pod. This can then blow through our allocated resource limits. Declare some modest amount of RAM and CPU resources and override `make` parallelism. This matches what rpm-ostree now does in https://github.com/coreos/rpm-ostree/pull/2155. --- .cci.jenkinsfile | 12 +++++++----- ci/libbuild.sh | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 4315a1d0..2a4c7288 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -2,7 +2,8 @@ stage("Build") { parallel normal: { - cosaPod(buildroot: true, runAsUser: 0) { + def n = 5 + cosaPod(buildroot: true, runAsUser: 0, memory: "2Gi", cpu: "${n}") { checkout scm stage("Core build") { shwrap(""" @@ -10,7 +11,7 @@ parallel normal: { git fetch origin --tags git submodule update --init - env SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} SKIP_INSTALLDEPS=1 ./ci/build.sh """) } stage("Unit tests") { @@ -84,18 +85,19 @@ parallel fcos: { } }, buildopts: { - cosaPod(buildroot: true, runAsUser: 0) { + def n = 5 + cosaPod(buildroot: true, runAsUser: 0, memory: "2Gi", cpu: "${n}") { checkout scm shwrap(""" git submodule update --init git worktree add build-rust && cd build-rust - env CONFIGOPTS="--enable-rust" SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} CONFIGOPTS="--enable-rust" SKIP_INSTALLDEPS=1 ./ci/build.sh make check TESTS=tests/test-rollsum cd .. && rm -rf build-rust git worktree add build-libsoup && cd build-libsoup - env CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh + env MAKE_JOBS=${n} CONFIGOPTS="--without-curl --without-openssl --with-soup" SKIP_INSTALLDEPS=1 ./ci/build.sh make check cd .. && rm -rf build-libsoup """) diff --git a/ci/libbuild.sh b/ci/libbuild.sh index a8ade0d7..dece8d09 100644 --- a/ci/libbuild.sh +++ b/ci/libbuild.sh @@ -10,7 +10,7 @@ pkg_upgrade() { } make() { - /usr/bin/make -j $(getconf _NPROCESSORS_ONLN) "$@" + /usr/bin/make -j ${MAKE_JOBS:-$(getconf _NPROCESSORS_ONLN)} "$@" } build() { From 6a5f97c14522f46f202f6cf9449c6e9fe0f4d15b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Jul 2020 01:36:51 +0000 Subject: [PATCH 174/177] tests: Add needs-internet tag for webserver bits Fixes the tests, see https://github.com/coreos/coreos-assembler/pull/1600 TODO: provide a webserver binary via virtio or so --- tests/kolainst/nondestructive/itest-payload-link.sh | 2 ++ tests/kolainst/nondestructive/itest-pull.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kolainst/nondestructive/itest-payload-link.sh b/tests/kolainst/nondestructive/itest-payload-link.sh index 6cfe291a..cbd82d41 100755 --- a/tests/kolainst/nondestructive/itest-payload-link.sh +++ b/tests/kolainst/nondestructive/itest-payload-link.sh @@ -1,4 +1,6 @@ #!/bin/bash +# FIXME just for webserver +# kola: { "tags": "needs-internet" } # # Copyright (C) 2018 Red Hat, Inc. # diff --git a/tests/kolainst/nondestructive/itest-pull.sh b/tests/kolainst/nondestructive/itest-pull.sh index 770f2444..5e8776aa 100755 --- a/tests/kolainst/nondestructive/itest-pull.sh +++ b/tests/kolainst/nondestructive/itest-pull.sh @@ -1,5 +1,6 @@ #!/bin/bash - +# FIXME just for webserver +# kola: { "tags": "needs-internet" } # Using the host ostree, test HTTP pulls set -xeuo pipefail From 5b75358357ee7710a8b6a8e20cc6242f5f16e6bd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jul 2020 14:20:07 +0000 Subject: [PATCH 175/177] pull: Also append bytes written This is very useful information that we get from the transaction stats. Append it to the final display if we're not inheriting the transaction. --- src/libostree/ostree-repo-pull.c | 8 +++++++- tests/pull-test.sh | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5a276e62..6d35be26 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3358,6 +3358,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, * value, otherwise NULL. Used for logging. */ const char *the_ref_to_fetch = NULL; + OstreeRepoTransactionStats tstats = { 0, }; /* Default */ pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE; @@ -4444,7 +4445,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } if (!inherit_transaction && - !ostree_repo_commit_transaction (pull_data->repo, NULL, cancellable, error)) + !ostree_repo_commit_transaction (pull_data->repo, &tstats, cancellable, error)) goto out; end_time = g_get_monotonic_time (); @@ -4486,6 +4487,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, shift == 1 ? "B" : "KiB", (guint) ((end_time - pull_data->start_time) / G_USEC_PER_SEC)); } + if (!inherit_transaction) + { + g_autofree char *bytes_written = g_format_size (tstats.content_bytes_written); + g_string_append_printf (buf, "; %s content written", bytes_written); + } ostree_async_progress_set_status (pull_data->progress, buf->str); } diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 92bcf112..07851a2c 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -64,7 +64,7 @@ fi # Try both syntaxes repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main >out.txt -assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched" +assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched; [1-9][0-9]*.*written" ${CMD_PREFIX} ostree --repo=repo pull origin:main > out.txt assert_not_file_has_content out.txt "[1-9][0-9]* content objects fetched" ${CMD_PREFIX} ostree --repo=repo fsck From a615d35762be74b6407bfcbdde5387f84dd151cf Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Jul 2020 21:13:36 +0000 Subject: [PATCH 176/177] pull: Add --per-object-fsync This is the opposite of https://github.com/ostreedev/ostree/issues/1184 Motivated by OpenShift seeing etcd performance issues during OS updates: https://github.com/openshift/machine-config-operator/issues/1897 Basically, if we switch to invoking `fsync()` as we go, it makes ostree performance worse (in my tests, 31s to write 2G versus 7s if we delay sync) but it avoids *huge* outliers in `fsync()` time for etcd. --- man/ostree.repo-config.xml | 12 +++ src/libostree/ostree-repo-commit.c | 113 ++++++++++++++++++---------- src/libostree/ostree-repo-private.h | 17 +++-- src/libostree/ostree-repo-pull.c | 7 ++ src/libostree/ostree-repo.c | 4 + src/ostree/ot-builtin-pull-local.c | 5 ++ src/ostree/ot-builtin-pull.c | 6 +- tests/pull-test.sh | 14 +++- 8 files changed, 131 insertions(+), 47 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index 90ac9083..7a01fc01 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -127,6 +127,18 @@ Boston, MA 02111-1307, USA. + + per-object-fsync + By default, OSTree will batch fsync() after + writing everything; however, this can cause latency spikes + for other processes which are also invoking fsync(). + Turn on this boolean to reduce potential latency spikes, + at the cost of slowing down OSTree updates. You most + likely want this on by default for "background" OS updates. + + + + min-free-space-percent diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 727e853b..0c9de239 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -51,15 +51,33 @@ #define FICLONE _IOW(0x94, 9, int) #endif - -/* If fsync is enabled and we're in a txn, we write into a staging dir for - * commit, but we also allow direct writes into objects/ for e.g. hardlink - * imports. +/* Understanding ostree's fsync strategy + * + * A long time ago, ostree used to invoke fsync() on each object, + * then move it into the objects directory. However, it turned + * out to be a *lot* faster to write the objects into a separate "staging" + * directory (letting the filesystem handle writeback how it likes) + * and then only walk over each of the files, fsync(), then rename() + * into place. See also https://lwn.net/Articles/789024/ + * + * (We also support a "disable fsync entirely" mode, where you don't + * care about integrity; e.g. test suites using disposable VMs). + * + * This "delayed fsync" pattern though is much worse for other concurrent processes + * like databases because it forces a lot to go through the filesystem + * journal at once once we do the sync. So now we support a `per_object_fsync` + * option that again invokes `fsync()` directly. This also notably + * provides "backpressure", ensuring we aren't queuing up a huge amount + * of I/O at once. */ + +/* The directory where we place content */ static int commit_dest_dfd (OstreeRepo *self) { - if (self->in_transaction && !self->disable_fsync) + if (self->per_object_fsync) + return self->objects_dir_fd; + else if (self->in_transaction && !self->disable_fsync) return self->commit_stagedir.fd; else return self->objects_dir_fd; @@ -420,7 +438,7 @@ commit_loose_regfile_object (OstreeRepo *self, /* Ensure that in case of a power cut, these files have the data we * want. See http://lwn.net/Articles/322823/ */ - if (!self->in_transaction && !self->disable_fsync) + if (!self->disable_fsync && self->per_object_fsync) { if (fsync (tmpf->fd) == -1) return glnx_throw_errno_prefix (error, "fsync"); @@ -1835,6 +1853,52 @@ ostree_repo_prepare_transaction (OstreeRepo *self, return TRUE; } +/* Synchronize the directories holding the objects */ +static gboolean +fsync_object_dirs (OstreeRepo *self, + GCancellable *cancellable, + GError **error) +{ + GLNX_AUTO_PREFIX_ERROR ("fsync objdirs", error); + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + + if (self->disable_fsync) + return TRUE; /* No fsync? Nothing to do then. */ + + if (!glnx_dirfd_iterator_init_at (self->objects_dir_fd, ".", FALSE, &dfd_iter, error)) + return FALSE; + while (TRUE) + { + struct dirent *dent; + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + if (dent->d_type != DT_DIR) + continue; + /* All object directories only have two character entries */ + if (strlen (dent->d_name) != 2) + continue; + + glnx_autofd int target_dir_fd = -1; + if (!glnx_opendirat (self->objects_dir_fd, dent->d_name, FALSE, + &target_dir_fd, error)) + return FALSE; + /* This synchronizes the directory to ensure all the objects we wrote + * are there. We need to do this before removing the .commitpartial + * stamp (or have a ref point to the commit). + */ + if (fsync (target_dir_fd) == -1) + return glnx_throw_errno_prefix (error, "fsync"); + } + + /* In case we created any loose object subdirs, make sure they are on disk */ + if (fsync (self->objects_dir_fd) == -1) + return glnx_throw_errno_prefix (error, "fsync"); + + return TRUE; +} + /* Called for commit, to iterate over the "staging" directory and rename all the * objects into the primary objects/ location. Notably this is called only after * syncfs() has potentially been invoked to ensure that all objects have been @@ -1856,10 +1920,6 @@ rename_pending_loose_objects (OstreeRepo *self, while (TRUE) { struct dirent *dent; - gboolean renamed_some_object = FALSE; - g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, }; - char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; - if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) return FALSE; if (dent == NULL) @@ -1872,10 +1932,12 @@ rename_pending_loose_objects (OstreeRepo *self, if (strlen (dent->d_name) != 2) continue; + g_auto(GLnxDirFdIterator) child_dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, &child_dfd_iter, error)) return FALSE; + char loose_objpath[_OSTREE_LOOSE_PATH_MAX]; loose_objpath[0] = dent->d_name[0]; loose_objpath[1] = dent->d_name[1]; loose_objpath[2] = '/'; @@ -1899,35 +1961,7 @@ rename_pending_loose_objects (OstreeRepo *self, if (!glnx_renameat (child_dfd_iter.fd, loose_objpath + 3, self->objects_dir_fd, loose_objpath, error)) return FALSE; - - renamed_some_object = TRUE; } - - if (renamed_some_object && !self->disable_fsync) - { - /* Ensure that in the case of a power cut all the directory metadata that - we want has reached the disk. In particular, we want this before we - update the refs to point to these objects. */ - glnx_autofd int target_dir_fd = -1; - - loose_objpath[2] = 0; - - if (!glnx_opendirat (self->objects_dir_fd, - loose_objpath, FALSE, - &target_dir_fd, - error)) - return FALSE; - - if (fsync (target_dir_fd) == -1) - return glnx_throw_errno_prefix (error, "fsync"); - } - } - - /* In case we created any loose object subdirs, make sure they are on disk */ - if (!self->disable_fsync) - { - if (fsync (self->objects_dir_fd) == -1) - return glnx_throw_errno_prefix (error, "fsync"); } return TRUE; @@ -2377,6 +2411,9 @@ ostree_repo_commit_transaction (OstreeRepo *self, if (!rename_pending_loose_objects (self, cancellable, error)) return FALSE; + if (!fsync_object_dirs (self, cancellable, error)) + return FALSE; + g_debug ("txn commit %s", glnx_basename (self->commit_stagedir.path)); if (!glnx_tmpdir_delete (&self->commit_stagedir, cancellable, error)) return FALSE; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a744c069..8c1f5071 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -38,13 +38,17 @@ G_BEGIN_DECLS #define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8 #define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2 -/* In most cases, writing to disk should be much faster than - * fetching from the network, so we shouldn't actually hit - * this. But if using pipelining and e.g. pulling over LAN - * (or writing to slow media), we can have a runaway - * situation towards EMFILE. +/* We want some parallelism with disk writes, but we also + * want to avoid starting tens or hundreds of threads + * (via GTask) all writing to disk. Eventually we may + * use io_uring which handles backpressure correctly. + * Also, in "immediate fsync" mode, this helps provide + * much more backpressure, helping our I/O patterns + * be nicer for any concurrent processes, such as etcd + * or other databases. + * https://github.com/openshift/machine-config-operator/issues/1897 * */ -#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16 +#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 3 /* Well-known keys for the additional metadata field in a summary file. */ #define OSTREE_SUMMARY_LAST_MODIFIED "ostree.summary.last-modified" @@ -147,6 +151,7 @@ struct OstreeRepo { GError *writable_error; gboolean in_transaction; gboolean disable_fsync; + gboolean per_object_fsync; gboolean disable_xattrs; guint zlib_compression_level; GHashTable *loose_object_devino_hash; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 5a276e62..2e9e6103 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -3286,6 +3286,7 @@ initiate_request (OtPullData *pull_data, * * disable-sign-verify (b): Disable signapi verification of commits * * disable-sign-verify-summary (b): Disable signapi verification of the summary * * depth (i): How far in the history to traverse; default is 0, -1 means infinite + * * per-object-fsync (b): Perform disk writes more slowly, avoiding a single large I/O sync * * disable-static-deltas (b): Do not use static deltas * * require-static-deltas (b): Require static deltas * * override-commit-ids (as): Array of specific commit IDs to fetch for refs @@ -3340,6 +3341,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_autoptr(GVariantIter) collection_refs_iter = NULL; g_autofree char **override_commit_ids = NULL; g_autoptr(GSource) update_timeout = NULL; + gboolean opt_per_object_fsync = FALSE; gboolean opt_gpg_verify_set = FALSE; gboolean opt_gpg_verify_summary_set = FALSE; gboolean opt_collection_refs_set = FALSE; @@ -3385,6 +3387,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas); (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); + (void) g_variant_lookup (options, "per-object-fsync", "b", &opt_per_object_fsync); (void) g_variant_lookup (options, "override-url", "&s", &url_override); (void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction); (void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers); @@ -3453,6 +3456,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->main_context = g_main_context_ref_thread_default (); pull_data->flags = flags; + /* TODO: Avoid mutating the repo object */ + if (opt_per_object_fsync) + self->per_object_fsync = TRUE; + if (!opt_n_network_retries_set) pull_data->n_network_retries = DEFAULT_N_NETWORK_RETRIES; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 82dd286a..95eb0efc 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2922,6 +2922,10 @@ reload_core_config (OstreeRepo *self, ostree_repo_set_disable_fsync (self, TRUE); } + if (!ot_keyfile_get_boolean_with_default (self->config, "core", "per-object-fsync", + FALSE, &self->per_object_fsync, error)) + return FALSE; + /* See https://github.com/ostreedev/ostree/issues/758 */ if (!ot_keyfile_get_boolean_with_default (self->config, "core", "disable-xattrs", FALSE, &self->disable_xattrs, error)) diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index c42d38d7..43f4f255 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -34,6 +34,7 @@ static char *opt_remote; static gboolean opt_commit_only; static gboolean opt_disable_fsync; +static gboolean opt_per_object_fsync; static gboolean opt_untrusted; static gboolean opt_bareuseronly_files; static gboolean opt_require_static_deltas; @@ -50,6 +51,7 @@ static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { "per-object-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_per_object_fsync, "Perform writes in such a way that avoids stalling concurrent processes", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Verify checksums of local sources (always enabled for HTTP pulls)", NULL }, { "bareuseronly-files", 0, 0, G_OPTION_ARG_NONE, &opt_bareuseronly_files, "Reject regular files with mode outside of 0775 (world writable, suid, etc.)", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, @@ -188,6 +190,9 @@ ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invoc g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "disable-sign-verify-summary", g_variant_new_variant (g_variant_new_boolean (TRUE))); + if (opt_per_object_fsync) + g_variant_builder_add (&builder, "{s@v}", "per-object-fsync", + g_variant_new_variant (g_variant_new_boolean (TRUE))); if (console.is_tty) progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, &console); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 1625ab47..e69d62e3 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -29,6 +29,7 @@ #include "otutil.h" static gboolean opt_disable_fsync; +static gboolean opt_per_object_fsync; static gboolean opt_mirror; static gboolean opt_commit_only; static gboolean opt_dry_run; @@ -58,6 +59,7 @@ static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "cache-dir", 0, 0, G_OPTION_ARG_FILENAME, &opt_cache_dir, "Use custom cache dir", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { "per-object-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_per_object_fsync, "Perform writes in such a way that avoids stalling concurrent processes", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror and fetches all refs if none provided", NULL }, @@ -325,7 +327,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeCommandInvocation *invocation, if (opt_localcache_repos) g_variant_builder_add (&builder, "{s@v}", "localcache-repos", g_variant_new_variant (g_variant_new_strv ((const char*const*)opt_localcache_repos, -1))); - + if (opt_per_object_fsync) + g_variant_builder_add (&builder, "{s@v}", "per-object-fsync", + g_variant_new_variant (g_variant_new_boolean (TRUE))); if (opt_http_headers) { GVariantBuilder hdr_builder; diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 92bcf112..615786e6 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -55,10 +55,10 @@ function verify_initial_contents() { } if has_gpgme; then - echo "1..34" + echo "1..35" else # 3 tests needs GPG support - echo "1..31" + echo "1..32" fi # Try both syntaxes @@ -74,6 +74,16 @@ cd ${test_tmpdir} verify_initial_contents echo "ok pull contents" +# And a test with incremental fsync +repo_init --no-sign-verify +${CMD_PREFIX} ostree --repo=repo pull --per-object-fsync origin main >out.txt +assert_file_has_content out.txt "[1-9][0-9]* metadata, [1-9][0-9]* content objects fetched" +${CMD_PREFIX} ostree --repo=repo pull --per-object-fsync origin:main > out.txt +assert_not_file_has_content out.txt "[1-9][0-9]* content objects fetched" +${CMD_PREFIX} ostree --repo=repo fsck +verify_initial_contents +echo "ok pull --per-object-fsync" + cd ${test_tmpdir} mkdir mirrorrepo ostree_repo_init mirrorrepo --mode=archive From 901747f98561cbeedb2ed150793d9b8f94451f20 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 21 Jul 2020 21:46:00 +0000 Subject: [PATCH 177/177] Release 2020.4 A lot of stuff here, new signing API is the biggest. Let's get a release out. --- configure.ac | 2 +- src/libostree/libostree-devel.sym | 22 ---------------------- src/libostree/libostree-released.sym | 22 ++++++++++++++++++++++ tests/test-symbols.sh | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/configure.ac b/configure.ac index f39a5006..855528c0 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ m4_define([year_version], [2020]) m4_define([release_version], [4]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 5999aabe..2ed658c4 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,28 +17,6 @@ Boston, MA 02111-1307, USA. ***/ -/* Add new symbols here. Release commits should copy this section into -released.sym. */ -LIBOSTREE_2020.2 { -global: - ostree_repo_commit_modifier_set_sepolicy_from_commit; - ostree_sign_get_type; - ostree_sign_get_all; - ostree_sign_commit; - ostree_sign_commit_verify; - ostree_sign_data; - ostree_sign_data_verify; - ostree_sign_get_by_name; - ostree_sign_get_name; - ostree_sign_clear_keys; - ostree_sign_load_pk; - ostree_sign_metadata_format; - ostree_sign_metadata_key; - ostree_sign_set_pk; - ostree_sign_add_pk; - ostree_sign_set_sk; - ostree_sign_summary; -} LIBOSTREE_2020.1; - /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste * source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION diff --git a/src/libostree/libostree-released.sym b/src/libostree/libostree-released.sym index 33d4d0e3..5c63f78f 100644 --- a/src/libostree/libostree-released.sym +++ b/src/libostree/libostree-released.sym @@ -593,6 +593,28 @@ global: ostree_sysroot_set_mount_namespace_in_use; } LIBOSTREE_2019.6; +/* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2020.4 { +global: + ostree_repo_commit_modifier_set_sepolicy_from_commit; + ostree_sign_get_type; + ostree_sign_get_all; + ostree_sign_commit; + ostree_sign_commit_verify; + ostree_sign_data; + ostree_sign_data_verify; + ostree_sign_get_by_name; + ostree_sign_get_name; + ostree_sign_clear_keys; + ostree_sign_load_pk; + ostree_sign_metadata_format; + ostree_sign_metadata_key; + ostree_sign_set_pk; + ostree_sign_add_pk; + ostree_sign_set_sk; + ostree_sign_summary; +} LIBOSTREE_2020.1; + /* No new symbols in 2020.2 */ /* NOTE: Only add more content here in release commits! See the diff --git a/tests/test-symbols.sh b/tests/test-symbols.sh index 8f67a46c..75df37e1 100755 --- a/tests/test-symbols.sh +++ b/tests/test-symbols.sh @@ -66,7 +66,7 @@ echo 'ok documented symbols' # ONLY update this checksum in release commits! cat > released-sha256.txt <