From e4e3e087eedde657c9593dfa01b7e449d1a1042f Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Tue, 23 Oct 2018 23:10:32 +0530 Subject: [PATCH 01/29] Post-release version bump Closes: #1761 Approved by: cgwalters --- configure.ac | 4 ++-- src/libostree/libostree-devel.sym | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 65e0e68c..8aeec3ab 100644 --- a/configure.ac +++ b/configure.ac @@ -4,10 +4,10 @@ dnl update libostree-released.sym from libostree-devel.sym, and update the check dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. m4_define([year_version], [2018]) -m4_define([release_version], [9]) +m4_define([release_version], [10]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=yes +is_release_build=no AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux]) diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index a4040ba6..4ff2b86f 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -18,6 +18,8 @@ ***/ /* Add new symbols here. Release commits should copy this section into -released.sym. */ +LIBOSTREE_2018.10 { +} LIBOSTREE_2018.9; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the last number. This is just a copy/paste From 759b099861a3065f01384d4d1121a7726771e5fe Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 24 Oct 2018 11:15:18 -0400 Subject: [PATCH 02/29] ostree/main: Wrap `Version` in `--version` in quotes Wrap the `Version` key in the YAML-compatible output of `ostree --version` with quotes so that it's parsed as a string. The issues with the previous approach in a nutshell: ``` In [5]: yaml.load("asdf: 2018.10") Out[5]: {'asdf': 2018.1} ``` It's treating the version number as a floating-point. Now, this is technically a backwards incompatible change, but given that the previous approach is inherently broken for our needs, I don't see a way around breaking it now. Closes: #1761 Approved by: cgwalters --- src/ostree/ot-main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 148c8371..a1449aef 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -354,7 +354,7 @@ ostree_option_context_parse (GOptionContext *context, * possible to parse */ g_auto(GStrv) features = g_strsplit (OSTREE_FEATURES, " ", -1); g_print ("%s:\n", PACKAGE_NAME); - g_print (" Version: %s\n", PACKAGE_VERSION); + g_print (" Version: '%s'\n", PACKAGE_VERSION); if (strlen (OSTREE_GITREV) > 0) g_print (" Git: %s\n", OSTREE_GITREV); #ifdef BUILDOPT_IS_DEVEL_BUILD From 62594765b2fae72e332ffb21fbd11085d911ef83 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 25 Oct 2018 15:15:04 +0000 Subject: [PATCH 03/29] build-sys: Dist ostree-finalize-staged.path Otherwise the tarball doesn't work. Closes: #1766 Approved by: jlebon --- Makefile-boot.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile-boot.am b/Makefile-boot.am index d18351af..b4d4a1af 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -67,6 +67,7 @@ EXTRA_DIST += src/boot/dracut/module-setup.sh \ src/boot/dracut/ostree.conf \ src/boot/mkinitcpio/ostree \ src/boot/ostree-prepare-root.service \ + src/boot/ostree-finalize-staged.path \ src/boot/ostree-remount.service \ src/boot/ostree-finalize-staged.service \ src/boot/grub2/grub2-15_ostree \ From 04da47a5fb9076b6a4f31ffa32d3608a420ab75f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 25 Oct 2018 15:15:51 +0000 Subject: [PATCH 04/29] ci: Add `dist-then-build` target to catch missing `EXTRA_DIST` I personally am very opposed to the entire idea of `make distcheck`; I think source code should canonically be *git* and not tarballs. See e.g. https://github.com/cgwalters/git-evtag for some rationale. But anyways we are uploading classic tarballs since today that what Debian/Fedora/etc consume sadly, so we need to test it. We explicitly skip `make distcheck` since we don't want to rerun the test suite. Closes: #1766 Approved by: jlebon --- .papr.yml | 1 + Makefile.am | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.papr.yml b/.papr.yml index f7044e5d..79843580 100644 --- a/.papr.yml +++ b/.papr.yml @@ -47,6 +47,7 @@ tests: - ci/ci-commitmessage-submodules.sh - ci/build-check.sh - ci/ci-release-build.sh + - make dist-then-build artifacts: - test-suite.log diff --git a/Makefile.am b/Makefile.am index 85542edb..4850cddd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -155,4 +155,14 @@ release-tarball-embedded: mv ostree-embeddeps-$${GITVERSION}.tar{.tmp,}; \ gzip -f ostree-embeddeps-$${GITVERSION}.tar +# `make dist` + `make`; explicitly not the other +# parts of distcheck like `make uninstall` since +# we don't care about that. +dist-then-build: dist + rm $(distdir) -rf && tar -xf $(distdir).tar.xz + cd $(distdir) && mkdir _build && cd _build && \ + ../configure --prefix=/usr --libdir=/usr/lib --sysconfdir=/etc && \ + $(MAKE) && make install DESTDIR=$$(pwd)/_install && \ + rm -rf $(distdir) + -include $(top_srcdir)/git.mk From 51752baf0e841d7a1843b4e77852878077dfe660 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Tue, 6 Mar 2018 00:01:14 +0100 Subject: [PATCH 05/29] checkout: honor opaque checkouts if a file ".wh..wh..opq" is present in a directory, delete anything from lower layers that is already in that directory. Signed-off-by: Giuseppe Scrivano Closes: #1486 Approved by: cgwalters --- src/libostree/ostree-repo-checkout.c | 25 +++++++++++++++++++++++++ tests/basic-test.sh | 3 ++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 5ae79923..e190a14a 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -35,6 +35,7 @@ #include "ostree-repo-private.h" #define WHITEOUT_PREFIX ".wh." +#define OPAQUE_WHITEOUT_NAME ".wh..wh..opq" /* Per-checkout call state/caching */ typedef struct { @@ -879,6 +880,7 @@ checkout_tree_at_recurse (OstreeRepo *self, GError **error) { gboolean did_exist = FALSE; + gboolean is_opaque_whiteout = FALSE; const gboolean sepolicy_enabled = options->sepolicy && !self->disable_xattrs; g_autoptr(GVariant) dirtree = NULL; g_autoptr(GVariant) dirmeta = NULL; @@ -912,6 +914,22 @@ checkout_tree_at_recurse (OstreeRepo *self, return TRUE; /* Note early return */ } + if (options->process_whiteouts) + { + g_autoptr(GVariant) dir_file_contents = g_variant_get_child_value (dirtree, 0); + GVariantIter viter; + const char *fname; + g_autoptr(GVariant) contents_csum_v = NULL; + g_variant_iter_init (&viter, dir_file_contents); + while (g_variant_iter_loop (&viter, "(&s@ay)", &fname, &contents_csum_v)) + { + is_opaque_whiteout = (g_str_equal (fname, OPAQUE_WHITEOUT_NAME)); + if (is_opaque_whiteout) + break; + } + contents_csum_v = NULL; /* iter_loop freed it */ + } + /* First, make the directory. Push a new scope in case we end up using * setfscreatecon(). */ @@ -931,6 +949,13 @@ checkout_tree_at_recurse (OstreeRepo *self, return FALSE; } + /* If it is an opaque whiteout, ensure the destination is empty first. */ + if (is_opaque_whiteout) + { + if (!glnx_shutil_rm_rf_at (destination_parent_fd, destination_name, cancellable, error)) + return FALSE; + } + /* Create initially with mode 0700, then chown/chmod only when we're * done. This avoids anyone else being able to operate on partially * constructed dirs. diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 3c4823d7..de4348c2 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -1034,7 +1034,7 @@ echo "ok test error pre commit/bootid" # Whiteouts cd ${test_tmpdir} mkdir -p overlay/baz/ -if touch overlay/baz/.wh.cow && touch overlay/.wh.deeper; then +if touch overlay/baz/.wh.cow && touch overlay/.wh.deeper && touch overlay/baz/another/.wh..wh..opq; then touch overlay/anewfile mkdir overlay/anewdir/ touch overlay/anewdir/blah @@ -1050,6 +1050,7 @@ if touch overlay/baz/.wh.cow && touch overlay/.wh.deeper; then assert_not_has_dir overlay-co/deeper assert_has_file overlay-co/anewdir/blah assert_has_file overlay-co/anewfile + assert_not_has_file overlay-co/baz/another/y # And test replacing a directory wholesale with a symlink as well as a regular file mkdir overlay From 34a8867749122550cd5d1b80ce22ba4340e00691 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 30 Oct 2018 13:21:46 -0400 Subject: [PATCH 06/29] bin/refs: Disallow creating broken aliases This is the alias version of #1749. I.e. we want to make sure that one can't even create an alias which would end up dangling. See also: https://pagure.io/releng/issue/7891 Closes: #1768 Approved by: sinnykumari --- src/ostree/ot-builtin-refs.c | 3 +++ tests/test-refs.sh | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-refs.c b/src/ostree/ot-builtin-refs.c index f88d08a6..630605a0 100644 --- a/src/ostree/ot-builtin-refs.c +++ b/src/ostree/ot-builtin-refs.c @@ -221,6 +221,9 @@ static gboolean do_ref (OstreeRepo *repo, const char *refspec_prefix, GCancellab { if (remote) return glnx_throw (error, "Cannot create alias to remote ref: %s", remote); + if (!g_hash_table_contains (refs, refspec_prefix)) + return glnx_throw (error, "Cannot create alias to non-existent ref: %s", + refspec_prefix); if (!ostree_repo_set_alias_ref_immediate (repo, remote, ref, refspec_prefix, cancellable, error)) goto out; diff --git a/tests/test-refs.sh b/tests/test-refs.sh index 1730423d..83da04ff 100755 --- a/tests/test-refs.sh +++ b/tests/test-refs.sh @@ -25,7 +25,7 @@ set -euo pipefail setup_fake_remote_repo1 "archive" -echo '1..6' +echo '1..7' cd ${test_tmpdir} mkdir repo @@ -207,3 +207,9 @@ if ${CMD_PREFIX} ostree --repo=repo refs -A exampleos/x86_64/27/server --create= fi assert_file_has_content_literal err.txt 'Cannot create alias to remote ref' echo "ok ref no alias remote" + +if ${CMD_PREFIX} ostree --repo=repo refs -A --create foobar nonexistent 2>err.txt; then + fatal "Created alias to nonexistent ref?" +fi +assert_file_has_content_literal err.txt 'Cannot create alias to non-existent ref' +echo "ok ref no broken alias" From 1dcebe8846335a1f306d599fb7172d84aaa7b0a0 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 31 Oct 2018 13:32:47 -0400 Subject: [PATCH 07/29] bin/pull-local: Add --commit-metadata-only It might be "local", but e.g. we may be crossing filesystems. So there are valid use cases for only wanting to pull the commit metadata with `pull-local`. Closes: #1769 Approved by: cgwalters --- src/ostree/ot-builtin-pull-local.c | 4 ++++ tests/basic-test.sh | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index b3e03bbb..4b3224f3 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -32,6 +32,7 @@ #include "otutil.h" static char *opt_remote; +static gboolean opt_commit_only; static gboolean opt_disable_fsync; static gboolean opt_untrusted; static gboolean opt_bareuseronly_files; @@ -46,6 +47,7 @@ static int opt_depth = 0; */ static GOptionEntry options[] = { + { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Verify checksums of local sources (always enabled for HTTP pulls)", NULL }, @@ -110,6 +112,8 @@ ostree_builtin_pull_local (int argc, char **argv, OstreeCommandInvocation *invoc pullflags |= OSTREE_REPO_PULL_FLAGS_UNTRUSTED; if (opt_bareuseronly_files) pullflags |= OSTREE_REPO_PULL_FLAGS_BAREUSERONLY_FILES; + if (opt_commit_only) + pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY; if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); diff --git a/tests/basic-test.sh b/tests/basic-test.sh index de4348c2..d112533e 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((85 + ${extra_basic_tests:-0}))" +echo "1..$((86 + ${extra_basic_tests:-0}))" CHECKOUT_U_ARG="" CHECKOUT_H_ARGS="-H" @@ -762,7 +762,19 @@ echo "ok subdir noent" if ! skip_one_without_user_xattrs; then cd ${test_tmpdir} - mkdir repo3 + mkdir repo4 + ostree_repo_init repo4 --mode=bare-user + ${CMD_PREFIX} ostree --repo=repo4 pull-local --commit-metadata-only repo test2 + csum1=$($OSTREE rev-parse test2) + csum2=$(${CMD_PREFIX} ostree --repo=repo4 rev-parse test2) + assert_streq "${csum1}" "${csum2}" + test -f repo4/state/$csum1.commitpartial + echo "ok pull-local --commit-metadata-only" + rm -rf repo4 +fi + +if ! skip_one_without_user_xattrs; then + cd ${test_tmpdir} ostree_repo_init repo3 --mode=bare-user ${CMD_PREFIX} ostree --repo=repo3 pull-local --remote=aremote repo test2 ${CMD_PREFIX} ostree --repo=repo3 rev-parse aremote/test2 From 8eac5be030797607715d5a2d973c55edff78a8a2 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 2 Nov 2018 15:39:34 -0400 Subject: [PATCH 08/29] lib/commit: Add devino_cache_hits to txn stats I found this useful while hacking on rpm-ostree but I think it might be useful enough to upstream. This stat is really helpful for validating that a pipeline is hitting the devino cache sweet spot. Closes: #1772 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 7 +++++++ src/libostree/ostree-repo.h | 3 ++- src/ostree/ot-builtin-commit.c | 1 + tests/test-basic-user.sh | 3 ++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 134024b8..52bc5d29 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3338,6 +3338,9 @@ write_content_to_mtree_internal (OstreeRepo *self, if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error)) return FALSE; } + g_mutex_lock (&self->txn_lock); + self->txn.stats.devino_cache_hits++; + g_mutex_unlock (&self->txn_lock); return TRUE; /* Early return */ } } @@ -3469,6 +3472,10 @@ write_content_to_mtree_internal (OstreeRepo *self, if (!ostree_mutable_tree_replace_file (mtree, name, loose_checksum, error)) return FALSE; + + g_mutex_lock (&self->txn_lock); + self->txn.stats.devino_cache_hits++; + g_mutex_unlock (&self->txn_lock); } /* Next fast path - we can "adopt" the file */ else if (can_adopt) diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 829164ba..007cbedd 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -291,8 +291,9 @@ struct _OstreeRepoTransactionStats { guint content_objects_total; guint content_objects_written; guint64 content_bytes_written; + guint devino_cache_hits; - guint64 padding1; + guint padding1; guint64 padding2; guint64 padding3; guint64 padding4; diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 535239be..b7c61a9e 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -835,6 +835,7 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio g_print ("Metadata Written: %u\n", stats.metadata_objects_written); g_print ("Content Total: %u\n", stats.content_objects_total); g_print ("Content Written: %u\n", stats.content_objects_written); + g_print ("Content Cache Hits: %u\n", stats.devino_cache_hits); g_print ("Content Bytes Written: %" G_GUINT64_FORMAT "\n", stats.content_bytes_written); } else diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index bd95f5ed..7bdb6a0c 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -117,9 +117,10 @@ assert_file_has_content ls.txt "^-006.. ${newuid} ${newgid} .*/baz/cow" # But --devino-canonical should override that $OSTREE commit ${COMMIT_ARGS} --owner-uid ${newuid} --owner-gid ${newgid} \ - -I -b test2-devino-test --tree=dir=test2-checkout + -I -b test2-devino-test --table-output --tree=dir=test2-checkout > out.txt $OSTREE ls test2-devino-test /baz/cow > ls.txt assert_file_has_content ls.txt "^-006.. ${myuid} ${mygid} .*/baz/cow" +assert_file_has_content out.txt "Content Cache Hits: [1-9][0-9]*" $OSTREE refs --delete test2-{linkcheckout,devino}-test echo "ok commit with -I" From 244d9a7ec1a08c0c7adf495a8b341798009556e1 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 31 Oct 2018 15:27:26 -0400 Subject: [PATCH 09/29] lib/commit: Copy user.ostreemeta only for bare-user When falling back to copying objects when importing them into a bare-user repo, we only actually need to transfer over the `user.ostreemeta` xattr. This allows the destination repo to be on a separate filesystem that might not even support `security.selinux`. (I hit this while importing over virtio-9p). Closes: #1771 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 52bc5d29..f8f8d070 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -4261,15 +4261,29 @@ import_one_object_direct (OstreeRepo *dest_repo, G_IN_SET (src_repo->mode, OSTREE_REPO_MODE_BARE, OSTREE_REPO_MODE_BARE_USER); if (src_is_bare_or_bare_user && !OSTREE_OBJECT_TYPE_IS_META(objtype)) { - g_autoptr(GVariant) xattrs = NULL; + if (src_repo == OSTREE_REPO_MODE_BARE) + { + g_autoptr(GVariant) xattrs = NULL; + if (!glnx_fd_get_all_xattrs (src_fd, &xattrs, + cancellable, error)) + return FALSE; + if (!glnx_fd_set_all_xattrs (tmp_dest.fd, xattrs, + cancellable, error)) + return FALSE; + } + else + { + /* bare-user; we just want ostree.usermeta */ + g_autoptr(GBytes) bytes = + glnx_fgetxattr_bytes (src_fd, "user.ostreemeta", error); + if (bytes == NULL) + return FALSE; - if (!glnx_fd_get_all_xattrs (src_fd, &xattrs, - cancellable, error)) - return FALSE; - - if (!glnx_fd_set_all_xattrs (tmp_dest.fd, xattrs, - cancellable, error)) - return FALSE; + if (TEMP_FAILURE_RETRY (fsetxattr (src_fd, "user.ostreemeta", + (char*)g_bytes_get_data (bytes, NULL), + g_bytes_get_size (bytes), 0)) != 0) + return glnx_throw_errno_prefix (error, "fsetxattr"); + } } if (fchmod (tmp_dest.fd, stbuf.st_mode & ~S_IFMT) != 0) From 05e8c7ef6a86e17a0ac421b9c80a2e57f56b4b3d Mon Sep 17 00:00:00 2001 From: rfairley Date: Tue, 6 Nov 2018 15:25:15 -0500 Subject: [PATCH 10/29] lib/repo: Search a list of paths in gpgkeypath for gpg keys This allows specifying gpgpath as list of paths that can point to a file or a directory. If a directory path is given, paths to all regular files in the directory are added to the remote as gpg ascii keys. If the path is not a directory, the file is directly added (whether regular file, empty - errors will be reported later when verifying gpg keys e.g. when pulling). Adding the gpgkeypath property looks like: ostree --repo=repo remote add --set=gpgpath="/path/key1.asc,/path/keys.d" R1 https://example.com/some/remote/ostree/repo Closes #773 Closes: #1773 Approved by: cgwalters --- man/ostree.xml | 37 +++++----- src/libostree/ostree-gpg-verifier.c | 73 +++++++++++++++++++- src/libostree/ostree-gpg-verifier.h | 13 ++++ src/libostree/ostree-repo.c | 16 +++-- src/libotutil/ot-keyfile-utils.c | 101 ++++++++++++++++++++++++++++ src/libotutil/ot-keyfile-utils.h | 17 +++++ tests/test-remote-gpg-import.sh | 73 ++++++++++++++++++++ 7 files changed, 307 insertions(+), 23 deletions(-) diff --git a/man/ostree.xml b/man/ostree.xml index d1c15665..f865982e 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -432,6 +432,7 @@ Boston, MA 02111-1307, USA. Examples + For specific examples, please see the man page regarding the specific ostree command. For example: @@ -445,28 +446,32 @@ Boston, MA 02111-1307, USA. OSTree supports signing commits with GPG. Operations on the system - repository by default use keyring files in + repository by default use keyring files in /usr/share/ostree/trusted.gpg.d. Any public key in a keyring file in that directory will be trusted by the client. No private keys should be present in this directory. - In addition to the system repository, OSTree supports two - other paths. First, there is a - gpgkeypath option for remotes, which must - point to the filename of an ASCII-armored key. - - Second, there is support for a per-remote - remotename.trustedkeys.gpg - file stored in the toplevel of the repository (alongside - objects/ and such). This is - particularly useful when downloading content that may not - be fully trusted (e.g. you want to inspect it but not - deploy it as an OS), or use it for containers. This file - is written via ostree remote add - --gpg-import. - + In addition to the system repository, OSTree supports two + other paths. First, there is a + gpgkeypath option for remotes, which must point + to the filename of an ASCII-armored GPG key, or a directory containing + ASCII-armored GPG keys to import. Multiple file and directory paths + to import from can be specified, as a comma-separated list of paths. This option + may be specified by using --set in ostree remote add. + + + Second, there is support for a per-remote + remotename.trustedkeys.gpg + file stored in the toplevel of the repository (alongside + objects/ and such). This is + particularly useful when downloading content that may not + be fully trusted (e.g. you want to inspect it but not + deploy it as an OS), or use it for containers. This file + is written via ostree remote add + --gpg-import. + diff --git a/src/libostree/ostree-gpg-verifier.c b/src/libostree/ostree-gpg-verifier.c index 1b70f8aa..a279348e 100644 --- a/src/libostree/ostree-gpg-verifier.c +++ b/src/libostree/ostree-gpg-verifier.c @@ -304,12 +304,82 @@ _ostree_gpg_verifier_add_key_ascii_file (OstreeGpgVerifier *self, g_ptr_array_add (self->key_ascii_files, g_strdup (path)); } +gboolean +_ostree_gpg_verifier_add_keyfile_path (OstreeGpgVerifier *self, + const char *path, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GError) temp_error = NULL; + if (!_ostree_gpg_verifier_add_keyfile_dir_at (self, AT_FDCWD, path, + cancellable, &temp_error)) + { + g_assert (temp_error); + + /* If failed due to not being a directory, add the file as an ascii key. */ + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_DIRECTORY)) + { + g_clear_error (&temp_error); + + _ostree_gpg_verifier_add_key_ascii_file (self, path); + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + + return FALSE; + } + } + return TRUE; +} + +/* Add files that exist one level below the directory at @path as ascii + * key files. If @path cannot be opened as a directory, + * an error is returned. + */ +gboolean +_ostree_gpg_verifier_add_keyfile_dir_at (OstreeGpgVerifier *self, + int dfd, + const char *path, + GCancellable *cancellable, + GError **error) +{ + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + + if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, + &dfd_iter, error)) + return FALSE; + + g_debug ("Adding GPG keyfile dir %s to verifier", path); + + while (TRUE) + { + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, + cancellable, error)) + return FALSE; + if (dent == NULL) + break; + + if (dent->d_type != DT_REG) + continue; + + /* TODO: Potentially open the files here and have the GPG verifier iterate + over the fds. See https://github.com/ostreedev/ostree/pull/1773#discussion_r235421900. */ + g_autofree char *iter_path = g_build_filename (path, dent->d_name, NULL); + + _ostree_gpg_verifier_add_key_ascii_file (self, iter_path); + } + + return TRUE; +} + gboolean _ostree_gpg_verifier_add_keyring_dir (OstreeGpgVerifier *self, GFile *path, GCancellable *cancellable, GError **error) - { return _ostree_gpg_verifier_add_keyring_dir_at (self, AT_FDCWD, gs_file_get_path_cached (path), @@ -322,7 +392,6 @@ _ostree_gpg_verifier_add_keyring_dir_at (OstreeGpgVerifier *self, const char *path, GCancellable *cancellable, GError **error) - { g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, diff --git a/src/libostree/ostree-gpg-verifier.h b/src/libostree/ostree-gpg-verifier.h index 4fe62d98..634d33b2 100644 --- a/src/libostree/ostree-gpg-verifier.h +++ b/src/libostree/ostree-gpg-verifier.h @@ -75,4 +75,17 @@ void _ostree_gpg_verifier_add_keyring_file (OstreeGpgVerifier *self, void _ostree_gpg_verifier_add_key_ascii_file (OstreeGpgVerifier *self, const char *path); +gboolean +_ostree_gpg_verifier_add_keyfile_path (OstreeGpgVerifier *self, + const char *path, + GCancellable *cancellable, + GError **error); + +gboolean +_ostree_gpg_verifier_add_keyfile_dir_at (OstreeGpgVerifier *self, + int dfd, + const char *path, + GCancellable *cancellable, + GError **error); + G_END_DECLS diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index fa3c2a94..f5231db9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5081,7 +5081,6 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, } else if (remote_name != NULL) { - g_autofree char *gpgkeypath = NULL; /* Add the remote's keyring file if it exists. */ g_autoptr(OstreeRemote) remote = NULL; @@ -5100,12 +5099,19 @@ _ostree_repo_gpg_verify_data_internal (OstreeRepo *self, add_global_keyring_dir = FALSE; } - if (!ot_keyfile_get_value_with_default (remote->options, remote->group, "gpgkeypath", NULL, - &gpgkeypath, error)) + g_auto(GStrv) gpgkeypath_list = NULL; + + if (!ot_keyfile_get_string_as_list (remote->options, remote->group, "gpgkeypath", + ";,", &gpgkeypath_list, error)) return NULL; - if (gpgkeypath) - _ostree_gpg_verifier_add_key_ascii_file (verifier, gpgkeypath); + if (gpgkeypath_list) + { + for (char **iter = gpgkeypath_list; *iter != NULL; ++iter) + if (!_ostree_gpg_verifier_add_keyfile_path (verifier, *iter, + cancellable, error)) + return NULL; + } } if (add_global_keyring_dir) diff --git a/src/libotutil/ot-keyfile-utils.c b/src/libotutil/ot-keyfile-utils.c index 3b29377f..a0ab75cc 100644 --- a/src/libotutil/ot-keyfile-utils.c +++ b/src/libotutil/ot-keyfile-utils.c @@ -101,6 +101,107 @@ ot_keyfile_get_value_with_default (GKeyFile *keyfile, return ret; } +/* Read the value of key as a string. If the value string contains + * one of the separators and none of the others, read the + * string as a NULL-terminated array out_value. If the value string contains + * none of the separators, read the string as a single entry into a + * NULL-terminated array out_value. If the value string contains multiple of + * the separators, an error is given. + * Returns TRUE on success, FALSE on error. */ +gboolean +ot_keyfile_get_string_as_list (GKeyFile *keyfile, + const char *section, + const char *key, + const char *separators, + char ***out_value, + GError **error) +{ + guint sep_count = 0; + gchar sep = '\0'; + g_autofree char *value_str = NULL; + g_autofree char **value_list = NULL; + + g_return_val_if_fail (keyfile != NULL, FALSE); + g_return_val_if_fail (section != NULL, FALSE); + g_return_val_if_fail (key != NULL, FALSE); + g_return_val_if_fail (separators != NULL, FALSE); + + if (!ot_keyfile_get_value_with_default (keyfile, section, key, NULL, + &value_str, error)) + return FALSE; + + if (value_str) + { + for (size_t i = 0; i < strlen (separators) && sep_count <= 1; i++) + { + if (strchr (value_str, separators[i])) + { + sep_count++; + sep = separators[i]; + } + } + + if (sep_count == 0) + { + value_list = g_new (gchar *, 2); + value_list[0] = g_steal_pointer (&value_str); + value_list[1] = NULL; + } + else if (sep_count == 1) + { + if (!ot_keyfile_get_string_list_with_default (keyfile, section, key, + sep, NULL, &value_list, error)) + return FALSE; + } + else + { + return glnx_throw (error, "key value list contains more than one separator"); + } + } + + ot_transfer_out_value (out_value, &value_list); + return TRUE; +} + +gboolean +ot_keyfile_get_string_list_with_default (GKeyFile *keyfile, + const char *section, + const char *key, + char separator, + char **default_value, + char ***out_value, + GError **error) +{ + g_autoptr(GError) temp_error = NULL; + + g_return_val_if_fail (keyfile != NULL, FALSE); + g_return_val_if_fail (section != NULL, FALSE); + g_return_val_if_fail (key != NULL, FALSE); + + g_key_file_set_list_separator (keyfile, separator); + + g_autofree char **ret_value = g_key_file_get_string_list (keyfile, section, + key, NULL, &temp_error); + + if (temp_error) + { + if (g_error_matches (temp_error, G_KEY_FILE_ERROR, + G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + { + g_clear_error (&temp_error); + ret_value = default_value; + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + return FALSE; + } + } + + ot_transfer_out_value (out_value, &ret_value); + return TRUE; +} + gboolean ot_keyfile_copy_group (GKeyFile *source_keyfile, GKeyFile *target_keyfile, diff --git a/src/libotutil/ot-keyfile-utils.h b/src/libotutil/ot-keyfile-utils.h index e136ee47..2dcb899c 100644 --- a/src/libotutil/ot-keyfile-utils.h +++ b/src/libotutil/ot-keyfile-utils.h @@ -44,6 +44,23 @@ ot_keyfile_get_value_with_default (GKeyFile *keyfile, char **out_value, GError **error); +gboolean +ot_keyfile_get_string_as_list (GKeyFile *keyfile, + const char *section, + const char *key, + const char *separators, + char ***out_value_list, + GError **error); + +gboolean +ot_keyfile_get_string_list_with_default (GKeyFile *keyfile, + const char *section, + const char *key, + char separator, + char **default_value, + char ***out_value, + GError **error); + gboolean ot_keyfile_copy_group (GKeyFile *source_keyfile, GKeyFile *target_keyfile, diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index f816429c..1bb42d82 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -154,6 +154,79 @@ ${OSTREE} prune --refs-only ${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key3.asc R4 $(cat httpd-address)/ostree/gnomerepo ${OSTREE} pull R4:main >/dev/null +# Test gpgkeypath success with multiple keys to try +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key1.asc,${test_tmpdir}/gpghome/key2.asc,${test_tmpdir}/gpghome/key3.asc R7 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R7:main >/dev/null + +# Test gpgkeypath failure with multiple keys but none in keyring +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key1.asc,${test_tmpdir}/gpghome/key2.asc R8 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R8:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with different key" +fi +assert_file_has_content err.txt "GPG signatures found, but none are in trusted keyring" + +# Test gpgkeypath success with directory containing a valid key +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/ R9 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R9:main >/dev/null + +# Test gpgkeypath failure with nonexistent directory +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/INVALIDKEYDIRPATH/ R10 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R10:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with nonexistent key directory" +fi +assert_file_has_content err.txt "INVALIDKEYDIRPATH.*No such file or directory" + +# Test gpgkeypath failure with a directory containing a valid key, and a nonexistent key +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/,${test_tmpdir}/gpghome/INVALIDKEYPATH.asc R11 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R11:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with nonexistent key" +fi +assert_file_has_content err.txt "INVALIDKEYPATH.*No such file or directory" + +# Test gpgkeypath success with a directory containing a valid key, and a key not in keyring +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/,${test_tmpdir}/gpghome/key1.asc R12 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R12:main >/dev/null + +# Test gpgkeypath failure with a nonexistent directory, and a valid key +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/INVALIDKEYDIRPATH/,${test_tmpdir}/gpghome/key3.asc R13 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R13:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with nonexistent key directory" +fi +assert_file_has_content err.txt "INVALIDKEYDIRPATH.*No such file or directory" + +# Test gpgkeypath failure with a nonexistent directory and a nonexistent key +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/INVALIDKEYDIRPATH/,${test_tmpdir}/gpghome/INVALIDKEYPATH.asc R14 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R14:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with nonexistent key" +fi +assert_file_has_content err.txt "INVALIDKEYDIRPATH.*No such file or directory" + +# Test gpgkeypath success for no trailing slash in directory path +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome R15 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R15:main >/dev/null + +# Test gpgkeypath failure with prefixed separator giving an empty path, and a nonexistent key +${OSTREE} remote add --set=gpgkeypath=,${test_tmpdir}/gpghome/INVALIDKEYPATH.asc R16 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R16:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with nonexistent key" +fi +assert_file_has_content err.txt "().*No such file or directory" + +# Test gpgkeypath success with suffixed separator +${OSTREE} remote add --set=gpgkeypath=${test_tmpdir}/gpghome/key3.asc, R17 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R17:main >/dev/null + +# Test gpgkeypath success with multiple keys specified, with semicolons +${OSTREE} remote add --set=gpgkeypath="${test_tmpdir}/gpghome/key1.asc;${test_tmpdir}/gpghome/key2.asc;${test_tmpdir}/gpghome/key3.asc" R18 $(cat httpd-address)/ostree/gnomerepo +${OSTREE} pull R18:main >/dev/null + +# Test gpgkeypath failure multiple keys specified, with mix of commas and semicolons +${OSTREE} remote add --set=gpgkeypath="${test_tmpdir}/gpghome/key1.asc,${test_tmpdir}/gpghome/key2.asc;${test_tmpdir}/gpghome/key3.asc" R19 $(cat httpd-address)/ostree/gnomerepo +if ${OSTREE} pull R19:main 2>err.txt; then + assert_not_reached "Unexpectedly succeeded at pulling with invalid gpgkeypath value" +fi +assert_file_has_content err.txt ".*key value list contains more than one separator" + rm repo/refs/remotes/* -rf ${OSTREE} prune --refs-only From c515c69d484f37be7d407dbcdaae10f6810b266b Mon Sep 17 00:00:00 2001 From: Matthew Leeds Date: Mon, 3 Dec 2018 15:27:39 -0800 Subject: [PATCH 11/29] lib/repo: Clarify docs for importing w/ trust The way _ostree_repo_import_object() is written, a hardlink copy is only attempted if the source repo is trusted, so update the docs for ostree_repo_import_object_from_with_trust() to reflect that. Closes: #1777 Approved by: cgwalters --- src/libostree/ostree-repo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index f5231db9..fc884481 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4214,9 +4214,9 @@ ostree_repo_import_object_from (OstreeRepo *self, * @error: Error * * Copy object named by @objtype and @checksum into @self from the - * source repository @source. If both repositories are of the same - * type and on the same filesystem, this will simply be a fast Unix - * hard link operation. + * source repository @source. If @trusted is %TRUE and both + * repositories are of the same type and on the same filesystem, + * this will simply be a fast Unix hard link operation. * * Otherwise, a copy will be performed. */ From dc2a7de2170aa35c33c9e767532f94be7f6f32c0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 4 Dec 2018 09:37:20 -0500 Subject: [PATCH 12/29] lib/commit: Try checksum+hardlink for untrusted local same-uid repos This mainly helps flatpak for enabling a hardlink-able local pull during deploy in the --system case. We assume the files are immutable when owned by the same uid. See https://github.com/ostreedev/ostree/issues/1723 and https://github.com/flatpak/flatpak/pull/2342 Closes: #1776 Approved by: uajain --- src/libostree/ostree-repo-commit.c | 36 ++++++++++++++---------- tests/basic-test.sh | 37 +++++++++++++++++++++++-- tests/test-basic-user-only.sh | 3 +- tests/test-basic-user.sh | 3 +- tests/test-basic.sh | 3 +- tests/test-pull-untrusted.sh | 44 +----------------------------- 6 files changed, 64 insertions(+), 62 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index f8f8d070..6cd3fcf4 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -4101,12 +4101,18 @@ import_is_bareuser_only_conversion (OstreeRepo *src_repo, && objtype == OSTREE_OBJECT_TYPE_FILE; } -/* Returns TRUE if we can potentially just call link() to copy an object. */ +/* Returns TRUE if we can potentially just call link() to copy an object; + * if untrusted the repos must be owned by the same uid. + */ static gboolean import_via_reflink_is_possible (OstreeRepo *src_repo, OstreeRepo *dest_repo, - OstreeObjectType objtype) + OstreeObjectType objtype, + gboolean trusted) { + /* Untrusted pulls require matching ownership */ + if (!trusted && (src_repo->owner_uid != dest_repo->owner_uid)) + return FALSE; /* Equal modes are always compatible, and metadata * is identical between all modes. */ @@ -4167,13 +4173,6 @@ import_one_object_direct (OstreeRepo *dest_repo, char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; _ostree_loose_path (loose_path_buf, checksum, objtype, dest_repo->mode); - if (!import_via_reflink_is_possible (src_repo, dest_repo, objtype)) - { - /* If we can't reflink, nothing to do here */ - *out_was_supported = FALSE; - return TRUE; - } - /* hardlinks require the owner to match and to be on the same device */ const gboolean can_hardlink = src_repo->owner_uid == dest_repo->owner_uid && @@ -4339,7 +4338,7 @@ _ostree_repo_import_object (OstreeRepo *self, */ const gboolean is_bareuseronly_conversion = import_is_bareuser_only_conversion (source, self, objtype); - gboolean try_direct = trusted; + gboolean try_direct = TRUE; /* If we need to do bareuseronly verification, or we're potentially doing a * bareuseronly conversion, let's verify those first so we don't complicate @@ -4379,11 +4378,20 @@ _ostree_repo_import_object (OstreeRepo *self, } } - /* We try to import via reflink/hardlink. If the remote is explicitly not trusted - * (i.e.) their checksums may be incorrect, we skip that. - */ - if (try_direct) + /* First, let's see if we can import via reflink/hardlink. */ + if (try_direct && import_via_reflink_is_possible (source, self, objtype, trusted)) { + /* For local repositories, if the untrusted flag is set, we verify the + * checksum first. This assumes then that the files are immutable - the + * above check verified that the owner uids match. + */ + if (!trusted) + { + if (!ostree_repo_fsck_object (source, objtype, checksum, + cancellable, error)) + return FALSE; + } + gboolean direct_was_supported = FALSE; if (!import_one_object_direct (self, source, checksum, objtype, &direct_was_supported, diff --git a/tests/basic-test.sh b/tests/basic-test.sh index d112533e..0c80a5f5 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -21,7 +21,7 @@ set -euo pipefail -echo "1..$((86 + ${extra_basic_tests:-0}))" +echo "1..$((88 + ${extra_basic_tests:-0}))" CHECKOUT_U_ARG="" CHECKOUT_H_ARGS="-H" @@ -364,6 +364,39 @@ if ! skip_one_without_user_xattrs; then echo "ok pull-local --bareuseronly-files" fi +rm repo2 -rf +ostree_repo_init repo2 --mode="$mode" +$CMD_PREFIX ostree --repo=repo2 pull-local --untrusted repo test2 +target_file_object=$(ostree_file_path_to_relative_object_path repo test2 baz/saucer) +target_file_checksum=$(ostree_file_path_to_checksum repo test2 baz/saucer) +assert_files_hardlinked repo{,2}/${target_file_object} +echo "ok pull-local hardlinking, untrusted" + +if grep -q 'mode=bare' repo/config; then + # Now copy/corrupt an object in a 3rd repo, pull into 2nd (leaving the first pristine) + rm repo{2,3} -rf + ostree_repo_init repo2 --mode="$mode" + ostree_repo_init repo3 --mode="$mode" + # Pull into 3rd repo, corrupt an object + $CMD_PREFIX ostree --repo=repo3 pull-local repo test2 + cp -a --reflink=auto repo3/${target_file_object}{,.tmp} + mv repo3/${target_file_object}{.tmp,} + echo blah >> repo3/${target_file_object} + if $CMD_PREFIX ostree --repo=repo2 pull-local --untrusted repo3 2>err.txt; then + assert_not_reached "pulled --untrusted from corrupted repo" + fi + assert_file_has_content err.txt 'Corrupted.*'${target_file_checksum} + rm -f err.txt + # But this one should succeed + $CMD_PREFIX ostree --repo=repo2 pull-local repo3 + if $CMD_PREFIX ostree --repo=repo2 fsck 2>err.txt; then + fatal "repo should have pulled corrupted object" + fi + assert_file_has_content err.txt 'Corrupted.*'${target_file_checksum} +fi +echo "ok pull-local --untrusted corruption" +rm repo{2,3} -rf + # This is mostly a copy of the suid test in test-basic-user-only.sh, # but for the `pull --bareuseronly-files` case. cd ${test_tmpdir} @@ -385,7 +418,7 @@ echo "ok pull-local (bareuseronly files)" if ! skip_one_without_user_xattrs; then cd ${test_tmpdir} - ${CMD_PREFIX} ostree --repo=repo2 checkout ${CHECKOUT_U_ARG} test2 test2-checkout-from-local-clone + ${CMD_PREFIX} ostree --repo=repo checkout ${CHECKOUT_U_ARG} test2 test2-checkout-from-local-clone cd test2-checkout-from-local-clone assert_file_has_content yet/another/tree/green 'leaf' echo "ok local clone checkout" diff --git a/tests/test-basic-user-only.sh b/tests/test-basic-user-only.sh index 5f27014e..f65094fd 100755 --- a/tests/test-basic-user-only.sh +++ b/tests/test-basic-user-only.sh @@ -23,7 +23,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh -setup_test_repository "bare-user-only" +mode="bare-user-only" +setup_test_repository "$mode" extra_basic_tests=5 . $(dirname $0)/basic-test.sh diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index 7bdb6a0c..e56f828a 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -25,7 +25,8 @@ set -euo pipefail skip_without_user_xattrs -setup_test_repository "bare-user" +mode="bare-user" +setup_test_repository "$mode" extra_basic_tests=6 . $(dirname $0)/basic-test.sh diff --git a/tests/test-basic.sh b/tests/test-basic.sh index c7f28c8c..b1bc37e5 100755 --- a/tests/test-basic.sh +++ b/tests/test-basic.sh @@ -25,5 +25,6 @@ set -euo pipefail skip_without_no_selinux_or_relabel -setup_test_repository "bare" +mode="bare" +setup_test_repository "$mode" . $(dirname $0)/basic-test.sh diff --git a/tests/test-pull-untrusted.sh b/tests/test-pull-untrusted.sh index 4c972599..ec48eb3e 100755 --- a/tests/test-pull-untrusted.sh +++ b/tests/test-pull-untrusted.sh @@ -25,52 +25,10 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo '1..4' +echo '1..1' setup_test_repository "bare" -cd ${test_tmpdir} -mkdir repo2 -ostree_repo_init repo2 --mode="bare" - -${CMD_PREFIX} ostree --repo=repo2 --untrusted pull-local repo - -find repo2 -type f -links +1 | while read line; do - assert_not_reached "pull-local created hardlinks" -done -echo "ok pull-local --untrusted didn't hardlink" - -# Corrupt repo -for i in ${test_tmpdir}/repo/objects/*/*.file; do - - # make sure it's not a symlink - if [ -L $i ]; then - continue - fi - - echo "corrupting $i" - echo "broke" >> $i - break; -done - -rm -rf repo2 -mkdir repo2 -ostree_repo_init repo2 --mode="bare" -if ${CMD_PREFIX} ostree --repo=repo2 pull-local repo; then - echo "ok trusted pull with corruption succeeded" -else - assert_not_reached "corrupted trusted pull unexpectedly succeeded!" -fi - -rm -rf repo2 -ostree_repo_init repo2 --mode="bare" -if ${CMD_PREFIX} ostree --repo=repo2 pull-local --untrusted repo; then - assert_not_reached "corrupted untrusted pull unexpectedly failed!" -else - echo "ok untrusted pull with corruption failed" -fi - - cd ${test_tmpdir} tar xf ${test_srcdir}/ostree-path-traverse.tar.gz rm -rf repo2 From cc2659e95789458e74b634827611e463d139d6c6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 7 Dec 2018 14:38:02 +0000 Subject: [PATCH 13/29] Update libglnx To pick up https://gitlab.gnome.org/GNOME/libglnx/merge_requests/4 Update submodule: libglnx Closes: #1781 Approved by: jlebon --- libglnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libglnx b/libglnx index 470af876..b1cb19b6 160000 --- a/libglnx +++ b/libglnx @@ -1 +1 @@ -Subproject commit 470af8763ff7b99bec950a6ae0a957c1dcfc8edd +Subproject commit b1cb19b6b2d712b492e6376248f3010d18e59daa From 30d79517ae0c539507d9044daa87626ac5467b6f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 7 Dec 2018 16:44:23 -0500 Subject: [PATCH 14/29] bin/pull: Add URL metavar for --url switch Otherwise it's not obvious that it takes an argument. Closes: #1782 Approved by: cgwalters --- src/ostree/ot-builtin-pull.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 8c6569cf..c68cbc46 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -66,7 +66,7 @@ static GOptionEntry options[] = { { "bareuseronly-files", 0, 0, G_OPTION_ARG_NONE, &opt_bareuseronly_files, "Reject regular files with mode outside of 0775 (world writable, suid, etc.)", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, - { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", NULL }, + { "url", 0, 0, G_OPTION_ARG_STRING, &opt_url, "Pull objects from this URL instead of the one from the remote config", "URL" }, { "http-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_http_headers, "Add NAME=VALUE as HTTP header to all requests", "NAME=VALUE" }, { "update-frequency", 0, 0, G_OPTION_ARG_INT, &opt_frequency, "Sets the update frequency, in milliseconds (0=1000ms) (default: 0)", "FREQUENCY" }, { "network-retries", 0, 0, G_OPTION_ARG_INT, &opt_network_retries, "Specifies how many times each download should be retried upon error (default: 5)", "N"}, From 416007e766eadc96e0591696b1aa5cd8dda6f5a4 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 8 Dec 2018 14:43:54 -0500 Subject: [PATCH 15/29] ci/papr: Drop insttests This didn't quite work out, and is now always failing because Ansible changed. For now we have some OK coverage via the rpm-ostree suite, let's just drop this and revisit later. Closes: #1783 Approved by: jlebon --- .papr.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.papr.yml b/.papr.yml index 79843580..7b3ca018 100644 --- a/.papr.yml +++ b/.papr.yml @@ -1,29 +1,3 @@ -# https://fedoraproject.org/wiki/CI/Tests -branches: - - master - - auto - - try - -context: FAH28-insttests -required: false - -# FIXME; temporary workaround -# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162 -host: - distro: fedora/28/atomic - specs: - ram: 4096 -#container: -# image: registry.fedoraproject.org/fedora:28 - -tests: - - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:28 /bin/sh -c "cd /srv/code && ./ci/fah28-insttests.sh" - -artifacts: - - tests/installed/artifacts/ - ---- - # This suite skips the RPMs and does the build+unit tests in a container inherit: false branches: From d044bfeb30b6a763487a3e9ccc2b3dc55dd9f92b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 10 Dec 2018 13:51:14 +0100 Subject: [PATCH 16/29] Fix leak in ostree_content_stream_parse() We need to ref_sink new GVariants for autoptr to work Closes: #1784 Approved by: cgwalters --- src/libostree/ostree-core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 080137f3..4e1a67e0 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -626,9 +626,9 @@ ostree_content_stream_parse (gboolean compressed, cancellable, error)) return FALSE; g_autoptr(GVariant) file_header = - g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT, - buf, archive_header_size, trusted, - g_free, buf); + g_variant_ref_sink(g_variant_new_from_data (compressed ? _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT : _OSTREE_FILE_HEADER_GVARIANT_FORMAT, + buf, archive_header_size, trusted, + g_free, buf)); buf = NULL; g_autoptr(GFileInfo) ret_file_info = NULL; g_autoptr(GVariant) ret_xattrs = NULL; From 3ecbdd81973dc706ddce1f28876533f81bb6889a Mon Sep 17 00:00:00 2001 From: Luca Bruno Date: Mon, 10 Dec 2018 16:03:36 +0000 Subject: [PATCH 17/29] lib/kargs: allow empty-list arguments This adds support for empty-list arguments (e.g. `acpi_osi=`), which are semantically different from simple-keyword arguments. Ref: https://github.com/projectatomic/rpm-ostree/issues/1706 Closes: #1785 Approved by: cgwalters --- src/libostree/ostree-kernel-args.c | 35 ++++++++++---------------- tests/test-admin-instutil-set-kargs.sh | 8 +++--- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/libostree/ostree-kernel-args.c b/src/libostree/ostree-kernel-args.c index a2575453..ac4014fa 100644 --- a/src/libostree/ostree-kernel-args.c +++ b/src/libostree/ostree-kernel-args.c @@ -33,23 +33,14 @@ static char * split_keyeq (char *arg) { char *eq; - + eq = strchr (arg, '='); - if (eq) - { - /* Note key/val are in one malloc block, - * so we don't free val... - */ - *eq = '\0'; - return eq+1; - } - else - { - /* ...and this allows us to insert a constant - * string. - */ - return ""; - } + if (eq == NULL) + return NULL; + + // Note: key/val are in a single allocation block, so we don't free val. + *eq = '\0'; + return eq+1; } static gboolean @@ -264,8 +255,10 @@ _ostree_kernel_args_to_strv (OstreeKernelArgs *kargs) for (j = 0; j < values->len; j++) { const char *value = values->pdata[j]; - - g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL)); + if (value == NULL) + g_ptr_array_add (strv, g_strconcat (key, NULL)); + else + g_ptr_array_add (strv, g_strconcat (key, "=", value, NULL)); } } g_ptr_array_add (strv, NULL); @@ -297,14 +290,12 @@ _ostree_kernel_args_to_string (OstreeKernelArgs *kargs) else g_string_append_c (buf, ' '); - if (value && *value) + g_string_append (buf, key); + if (value != NULL) { - g_string_append (buf, key); g_string_append_c (buf, '='); g_string_append (buf, value); } - else - g_string_append (buf, key); } } diff --git a/tests/test-admin-instutil-set-kargs.sh b/tests/test-admin-instutil-set-kargs.sh index 5568f238..1cb32189 100755 --- a/tests/test-admin-instutil-set-kargs.sh +++ b/tests/test-admin-instutil-set-kargs.sh @@ -33,21 +33,21 @@ ${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --remote=testos testo ${CMD_PREFIX} ostree admin deploy --os=testos testos:testos/buildmaster/x86_64-runtime ${CMD_PREFIX} ostree admin instutil set-kargs FOO=BAR -${CMD_PREFIX} ostree admin instutil set-kargs FOO=BAZ FOO=BIF TESTARG=TESTVALUE +${CMD_PREFIX} ostree admin instutil set-kargs FOO=BAZ FOO=BIF TESTARG=TESTVALUE KEYWORD EMPTYLIST= assert_not_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*FOO=BAR' assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*FOO=BAZ .*FOO=BIF' -assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE' +assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE KEYWORD EMPTYLIST=' echo "ok instutil set-kargs (basic)" ${CMD_PREFIX} ostree admin instutil set-kargs --merge FOO=BAR assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*FOO=BAZ .*FOO=BIF .*FOO=BAR' -assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE' +assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE KEYWORD EMPTYLIST=' echo "ok instutil set-kargs --merge" ${CMD_PREFIX} ostree admin instutil set-kargs --merge --replace=FOO=XXX assert_not_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*FOO=BAR' assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*FOO=XXX' -assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE' +assert_file_has_content sysroot/boot/loader/entries/ostree-1-testos.conf 'options.*TESTARG=TESTVALUE KEYWORD EMPTYLIST=' echo "ok instutil set-kargs --replace" ${CMD_PREFIX} ostree admin instutil set-kargs --merge --append=FOO=BAR --append=APPENDARG=VALAPPEND --append=APPENDARG=2NDAPPEND testos:testos/buildmaster/x86_64-runtime From 963395217aa4db4e22bd941aeef675464588d708 Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Thu, 6 Dec 2018 04:26:22 +0530 Subject: [PATCH 18/29] lib/repo-commit: Relax min-free-space check in prepare_transaction() We want a case where we can disable the min-free-space check. Initially, it felt like to add a OSTREE_REPO_PULL_FLAGS_DISABLE_FREE_SPACE_CHECK but the problem is prepare_transaction() does not have a OstreeRepoPullFlags parameter which we can parse right here. On top of it, prepare_transaction() enforces min-free-space check and won't let the transaction proceed if the check failed. This is pretty bad in conjunction with "inherit-transaction" as what Flatpak uses. There is no way to disable this check unless we remove it altogether from prepare_transaction. This issue came out to light when flatpak wasn't able to write metadata after fetching from remote: [uajain@localhost ~]$ flatpak remote-info flathub org.kde.Platform//5.9 error: min-free-space-size 500MB would be exceeded Metadata objects helps in housekeeping and restricting them means restricting crucial UX (like search, new updates) functionalities in clients like gnome-software. The error banners originated from these issues are also abrupt and not much helpful to the user. This is the specific instance of the issue this patches tries to address. See https://github.com/flatpak/flatpak/issues/2139 for discussion. Closes: #1779 Approved by: mwleeds --- src/libostree/ostree-repo-commit.c | 15 ++++++++++++--- .../installed/nondestructive/itest-pull-space.sh | 11 +++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 6cd3fcf4..533cf03a 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -1624,9 +1624,18 @@ ostree_repo_prepare_transaction (OstreeRepo *self, self->txn.max_blocks = bfree - self->reserved_blocks; else { - self->cleanup_stagedir = TRUE; - g_mutex_unlock (&self->txn_lock); - return throw_min_free_space_error (self, 0, error); + self->txn.max_blocks = 0; + /* Don't throw_min_free_space_error here; reason being that + * this transaction could be just committing metadata objects + * which are relatively small in size and we do not really + * want to block them via min-free-space-* value. Metadata + * objects helps in housekeeping and hence should be kept + * out of the strict min-free-space values. + * + * The main drivers for writing content objects will always honor + * the min-free-space value and throw_min_free_space_error in + * case of overstepping the number of reserved blocks. + */ } g_mutex_unlock (&self->txn_lock); diff --git a/tests/installed/nondestructive/itest-pull-space.sh b/tests/installed/nondestructive/itest-pull-space.sh index baf1042f..e365b86d 100755 --- a/tests/installed/nondestructive/itest-pull-space.sh +++ b/tests/installed/nondestructive/itest-pull-space.sh @@ -45,6 +45,17 @@ echo 'min-free-space-size=1MB' >> repo/config ostree --repo=repo pull-local /ostree/repo ${host_commit} echo "ok min-free-space-size (success)" +# metadata object write should succeed even if free space is lower than +# min-free-space value +rm -rf mnt/repo +ostree --repo=mnt/repo init --mode=bare-user +echo 'fsync=false' >> mnt/repo/config +echo 'min-free-space-size=10MB' >> mnt/repo/config +if ostree --repo=mnt/repo pull-local --commit-metadata-only /ostree/repo ${host_commit} 2>err.txt; then + fatal "could not write metadata objects even when min-free-space value should allow it" +fi +echo "ok metadata write even when free space is lower than min-free-space value" + rm -rf mnt/repo # Test min-free-space-size on deltas From c25b1c7063278a803c707c80bfc9006d96654a9e Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Wed, 12 Dec 2018 13:35:26 +0530 Subject: [PATCH 19/29] man/ostree.repo-config: Update min-free-space-* docs Docs should reflect the changes done in PR: https://github.com/ostreedev/ostree/pull/1779 Closes: #1786 Approved by: jlebon --- man/ostree.repo-config.xml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index acaa3be5..618b524b 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -138,7 +138,11 @@ Boston, MA 02111-1307, USA. If min-free-space-size is set to a non-zero - value, min-free-space-percent is ignored. + value, min-free-space-percent is ignored. Note + that, min-free-space-percent is not enforced on + metadata objects. It is assumed that metadata objects are relatively + small in size compared to content objects and thus kept outside the + scope of this option. @@ -156,7 +160,10 @@ Boston, MA 02111-1307, USA. If this option is set to a non-zero value, and min-free-space-percent is also set, this option - takes priority. + takes priority. Note that, min-free-space-size is + not enforced on metadata objects. It is assumed that metadata objects + are relatively small in size compared to content objects and thus kept + outside the scope of this option. From f2d4537ffb5c0ebdbdb37bc031940f8e8c0631c6 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Thu, 13 Dec 2018 16:43:35 +0100 Subject: [PATCH 20/29] admin/status: Correctly check GPG signature Closes: #1770 Closes: #1787 Approved by: jlebon --- src/ostree/ot-admin-builtin-status.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-admin-builtin-status.c b/src/ostree/ot-admin-builtin-status.c index ca165f45..e625a927 100644 --- a/src/ostree/ot-admin-builtin-status.c +++ b/src/ostree/ot-admin-builtin-status.c @@ -141,10 +141,11 @@ deployment_print_status (OstreeSysroot *sysroot, g_autoptr(GString) output_buffer = g_string_sized_new (256); /* Print any digital signatures on this commit. */ + const char *osname = ostree_deployment_get_osname (deployment); g_autoptr(GError) local_error = NULL; g_autoptr(OstreeGpgVerifyResult) result = - ostree_repo_verify_commit_ext (repo, ref, NULL, NULL, - cancellable, &local_error); + ostree_repo_verify_commit_for_remote (repo, ref, osname, + cancellable, &local_error); /* G_IO_ERROR_NOT_FOUND just means the commit is not signed. */ if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) From c62232c09df203320012b63095063e7fe8231555 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 20 Dec 2018 09:21:20 -0500 Subject: [PATCH 21/29] boot/grub2: Add some more explanation in ostree-grub-generator Even with the previous docstring, I didn't understand at first the relationship between the `ostree-grub-generator` script and `ostree-bootloader-grub2.c`. Throw some more docs to clarify things a bit. Closes: #1791 Approved by: cgwalters --- src/boot/grub2/ostree-grub-generator | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/boot/grub2/ostree-grub-generator b/src/boot/grub2/ostree-grub-generator index 5b7ea1ab..2208c1bf 100644 --- a/src/boot/grub2/ostree-grub-generator +++ b/src/boot/grub2/ostree-grub-generator @@ -1,11 +1,12 @@ #!/bin/sh -# The builtin grub.cfg generator. +# The builtin grub.cfg generator. This script is called by +# ostree/src/libostree/ostree-bootloader-grub2.c whenever boot loader +# configuration file needs to be updated on systems which do not use +# grub2-mkconfig (and thus, the `ostree admin instutil grub2-generate` path). # -# This script is called by ostree/src/libostree/ostree-bootloader-grub2.c whenever -# boot loader configuration file needs to be updated. It can be used as a template -# for a custom grub.cfg generator. What to consider when writing a custom grub.cfg -# generator: +# It can be used as a template for a custom grub.cfg generator. What to consider +# when writing a custom grub.cfg generator: # # - The populate_menu() function converts boot loader entries as defined by # https://www.freedesktop.org/wiki/Specifications/BootLoaderSpec/ into GRUB2 From f0719681f90b272c089440f436f358fad79c4e06 Mon Sep 17 00:00:00 2001 From: Matthias Clasen Date: Sat, 15 Dec 2018 11:15:29 -0500 Subject: [PATCH 22/29] Fix a small leak in ostree_repo_pull_with_options Spotted in a flatpak valgrind log. Closes: #1788 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 1efb38b3..b7eba9d2 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -4643,6 +4643,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_object (&pull_data->cancellable); g_clear_pointer (&pull_data->localcache_repos, (GDestroyNotify)g_ptr_array_unref); g_clear_object (&pull_data->remote_repo_local); + g_free (pull_data->remote_refspec_name); g_free (pull_data->remote_name); g_free (pull_data->append_user_agent); g_clear_pointer (&pull_data->meta_mirrorlist, (GDestroyNotify) g_ptr_array_unref); From ee988f69aa56bab011ef55f5d561ebec23531ce4 Mon Sep 17 00:00:00 2001 From: Matthias Clasen Date: Sat, 15 Dec 2018 15:19:46 -0500 Subject: [PATCH 23/29] Fix a small leak in ostree_repo_remote_list_collection_refs g_variant_iter_loop would free the value for us, but not if we're returning from the middle of the loop body. Closes: #1788 Approved by: cgwalters --- src/libostree/ostree-repo-refs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 1bbe3901..1267b240 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -1002,7 +1002,7 @@ ostree_repo_remote_list_collection_refs (OstreeRepo *self, summary_collection_map = NULL; while (summary_collection_map != NULL && - g_variant_iter_loop (summary_collection_map, "{s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs)) + g_variant_iter_loop (summary_collection_map, "{&s@a(s(taya{sv}))}", &summary_collection_id, &summary_refs)) { if (!remote_list_collection_refs_process_refs (self, remote_name, summary_collection_id, summary_refs, From b6795c93ead221ce723238d8bcc94cb6621c8867 Mon Sep 17 00:00:00 2001 From: Ricardo Salveti Date: Mon, 17 Dec 2018 16:17:51 -0200 Subject: [PATCH 24/29] grub2: add support for devicetree Similar as available for u-boot (ce2995e1dc1557c4d97ef5af807eacf3ef4a22d8) and syslinux (c5112c25e4519835c4cd53f4350c1b2f2a477746), enable parsing and writing devicetree filename into grub.cfg. This is required by arm64-based devices running edk2 instead of u-boot as the main bootloader (e.g. 96boards HiKey and HiKey960). Signed-off-by: Ricardo Salveti Closes: #1790 Approved by: cgwalters --- src/boot/grub2/ostree-grub-generator | 7 +++++++ src/libostree/ostree-bootloader-grub2.c | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/boot/grub2/ostree-grub-generator b/src/boot/grub2/ostree-grub-generator index 2208c1bf..10645c74 100644 --- a/src/boot/grub2/ostree-grub-generator +++ b/src/boot/grub2/ostree-grub-generator @@ -33,6 +33,7 @@ read_config() initrd="" options="" linux="" + devicetree="" while read -r line do @@ -48,6 +49,9 @@ read_config() "linux") linux=${value} ;; + "devicetree") + devicetree=${value} + ;; "options") options=${value} ;; @@ -74,6 +78,9 @@ populate_menu() if [ -n "${initrd}" ] ; then menu="${menu}\t initrd ${boot_prefix}${initrd}\n" fi + if [ -n "${devicetree}" ] ; then + menu="${menu}\t devicetree ${boot_prefix}${devicetree}\n" + fi menu="${menu}}\n\n" done # The printf command seems to be more reliable across shells for special character (\n, \t) evaluation diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 6831382c..57673c7d 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -191,6 +191,7 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot const char *options; const char *kernel; const char *initrd; + const char *devicetree; char *quoted_title = NULL; char *uuid = NULL; char *quoted_uuid = NULL; @@ -246,6 +247,15 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot g_string_append_c (output, '\n'); } + devicetree = ostree_bootconfig_parser_get (config, "devicetree"); + if (devicetree) + { + g_string_append (output, "devicetree"); + g_string_append_c (output, ' '); + g_string_append (output, devicetree); + g_string_append_c (output, '\n'); + } + g_string_append (output, "}\n"); } From cd9138d9a09a14b1cf5ecc189c93e01dcae5f45e Mon Sep 17 00:00:00 2001 From: Sinny Kumari Date: Wed, 2 Jan 2019 16:03:19 +0530 Subject: [PATCH 25/29] atomic-upgrades doc: fix example url to fetch a ref checksum Signed-off-by: Sinny Kumari Closes: #1795 Approved by: cgwalters --- docs/manual/atomic-upgrades.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md index 9dbe6f8f..f2c01cc1 100644 --- a/docs/manual/atomic-upgrades.md +++ b/docs/manual/atomic-upgrades.md @@ -18,7 +18,7 @@ implements this. To begin a simple upgrade, OSTree fetches the contents of the ref from the remote server. Suppose we're tracking a ref named `exampleos/buildmaster/x86_64-runtime`. OSTree fetches the URL -`http://example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`, +`http://example.com/repo/refs/heads/exampleos/buildmaster/x86_64-runtime`, which contains a SHA256 checksum. This determines the tree to deploy, and `/etc` will be merged from currently booted tree. From 6e8377cbd1dc38564849523d0b5290fc22a3ada7 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Mon, 7 Jan 2019 15:12:42 +0100 Subject: [PATCH 26/29] admin/status: test GPG signature check Signed-off-by: Rafael Fonseca Closes: #1797 Approved by: cgwalters --- Makefile-tests.am | 1 + tests/test-admin-gpg.sh | 149 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100755 tests/test-admin-gpg.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 9837e5cd..417a304a 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -109,6 +109,7 @@ _installed_or_uninstalled_test_scripts = \ tests/test-admin-pull-deploy-split.sh \ tests/test-admin-locking.sh \ tests/test-admin-deploy-clean.sh \ + tests/test-admin-gpg.sh \ tests/test-reset-nonlinear.sh \ tests/test-oldstyle-partial.sh \ tests/test-delta.sh \ diff --git a/tests/test-admin-gpg.sh b/tests/test-admin-gpg.sh new file mode 100755 index 00000000..1f50c8a9 --- /dev/null +++ b/tests/test-admin-gpg.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# +# Copyright (C) 2019 Rafael Fonseca +# +# SPDX-License-Identifier: LGPL-2.0+ +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_os_repository_signed () { + mode=$1 + shift + bootmode=$1 + shift + bootdir=${1:-usr/lib/modules/3.6.0} + + oldpwd=`pwd` + keyid="472CDAFA" + + cd ${test_tmpdir} + mkdir testos-repo + if test -n "$mode"; then + ostree_repo_init testos-repo --mode=${mode} + else + ostree_repo_init testos-repo + fi + + cd ${test_tmpdir} + mkdir osdata + cd osdata + kver=3.6.0 + mkdir -p usr/bin ${bootdir} usr/lib/modules/${kver} usr/share usr/etc + kernel_path=${bootdir}/vmlinuz + initramfs_path=${bootdir}/initramfs.img + # /usr/lib/modules just uses "vmlinuz", since the version is in the module + # directory name. + if [[ $bootdir != usr/lib/modules/* ]]; then + kernel_path=${kernel_path}-${kver} + initramfs_path=${bootdir}/initramfs-${kver}.img + fi + echo "a kernel" > ${kernel_path} + echo "an initramfs" > ${initramfs_path} + bootcsum=$(cat ${kernel_path} ${initramfs_path} | sha256sum | cut -f 1 -d ' ') + export bootcsum + # Add the checksum for legacy dirs (/boot, /usr/lib/ostree-boot), but not + # /usr/lib/modules. + if [[ $bootdir != usr/lib/modules/* ]]; then + mv ${kernel_path}{,-${bootcsum}} + mv ${initramfs_path}{,-${bootcsum}} + fi + + echo "an executable" > usr/bin/sh + echo "some shared data" > usr/share/langs.txt + echo "a library" > usr/lib/libfoo.so.0 + ln -s usr/bin bin +cat > usr/etc/os-release < usr/etc/aconfigfile + mkdir -p usr/etc/NetworkManager + echo "a default daemon file" > usr/etc/NetworkManager/nm.conf + mkdir -p usr/etc/testdirectory + echo "a default daemon file" > usr/etc/testdirectory/test + + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string version=1.0.9 -b testos/buildmaster/x86_64-runtime -s "Build" --gpg-sign=$keyid --gpg-homedir=${test_tmpdir}/gpghome + + # Ensure these commits have distinct second timestamps + sleep 2 + echo "a new executable" > usr/bin/sh + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string version=1.0.10 -b testos/buildmaster/x86_64-runtime -s "Build" --gpg-sign=$keyid --gpg-homedir=${test_tmpdir}/gpghome + + cd ${test_tmpdir} + cp -a osdata osdata-devel + cd osdata-devel + mkdir -p usr/include + echo "a development header" > usr/include/foo.h + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string version=1.0.9 -b testos/buildmaster/x86_64-devel -s "Build" --gpg-sign=$keyid --gpg-homedir=${test_tmpdir}/gpghome + + ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo fsck -q + + cd ${test_tmpdir} + mkdir sysroot + export OSTREE_SYSROOT=sysroot + ${CMD_PREFIX} ostree admin init-fs sysroot + if test -n "${OSTREE_NO_XATTRS:-}"; then + echo -e 'disable-xattrs=true\n' >> sysroot/ostree/repo/config + fi + ${CMD_PREFIX} ostree admin os-init testos + + case $bootmode in + "syslinux") + setup_os_boot_syslinux + ;; + "uboot") + setup_os_boot_uboot + ;; + *grub2*) + setup_os_boot_grub2 "${bootmode}" + ;; + esac + + cd ${test_tmpdir} + mkdir ${test_tmpdir}/httpd + cd httpd + ln -s ${test_tmpdir} ostree + ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port + port=$(cat ${test_tmpdir}/httpd-port) + echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address + cd ${oldpwd} +} + +# Exports OSTREE_SYSROOT so --sysroot not needed. +setup_os_repository_signed "archive" "syslinux" + +echo "1..2" + +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo remote add testos $(cat httpd-address)/ostree/testos-repo +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo pull-local --gpg-verify=true --remote=testos testos-repo testos/buildmaster/x86_64-runtime +# This initial deployment gets kicked off with some kernel arguments +${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime +assert_has_dir sysroot/boot/ostree/testos-${bootcsum} + +echo "ok deploy command" + +${CMD_PREFIX} ostree admin status > status.txt +test -f status.txt +assert_file_has_content status.txt "GPG: Signature made" +assert_not_file_has_content status.txt "GPG: Can't check signature: public key not found" +echo 'ok gpg signature' From fa35ec38b774b639e0490a12ca22a425b517a6c9 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 8 Jan 2019 15:24:10 +0000 Subject: [PATCH 27/29] fixup! admin/status: test GPG signature check --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8aeec3ab..2fe51589 100644 --- a/configure.ac +++ b/configure.ac @@ -141,7 +141,7 @@ AS_HELP_STRING([--disable-http2], [Disable use of http2 (default: no)]),, [enable_http2=yes]) AS_IF([test x$enable_http2 != xno ], [ - AC_DEFINE([BUILDOPT_HTTP2], 1, [Define if we enable http2]) + AC_DEFINE([BUILDOPT_HTTP2], 1, [Define if we enable http2 by default]) ], [ OSTREE_FEATURES="$OSTREE_FEATURES no-http2" ]) From 308d30d0801401a0d417feb7d9ed793d668d6bfa Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 8 Jan 2019 09:41:26 -0500 Subject: [PATCH 28/29] pull: If built with --disable-http2, allow enabling via http2=1 We have a `http2=[0|1]` remote config option; let's have the `--disable-http2` build option define the default for that. This way it's easy to still enable http2 for testing even if we have it disabled by default. Closes: #1798 Approved by: jlebon --- src/libostree/ostree-fetcher-curl.c | 3 +-- src/libostree/ostree-repo-pull.c | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c index 9738f980..294b2078 100644 --- a/src/libostree/ostree-fetcher-curl.c +++ b/src/libostree/ostree-fetcher-curl.c @@ -777,7 +777,6 @@ initiate_next_curl_request (FetcherRequest *req, * there are numerous HTTP/2 fixes since the original version in * libcurl 7.43.0. */ -#ifdef BUILDOPT_HTTP2 if (!(self->config_flags & OSTREE_FETCHER_FLAGS_DISABLE_HTTP2)) { #if CURL_AT_LEAST_VERSION(7, 51, 0) @@ -789,7 +788,7 @@ initiate_next_curl_request (FetcherRequest *req, curl_easy_setopt (req->easy, CURLOPT_PIPEWAIT, 1L); #endif } -#endif + curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb); if (g_getenv ("OSTREE_DEBUG_HTTP")) curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L); diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index b7eba9d2..4b596da1 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2948,9 +2948,14 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, if (gzip) fetcher_flags |= OSTREE_FETCHER_FLAGS_TRANSFER_GZIP; - { gboolean http2 = TRUE; + { gboolean http2_default = TRUE; +#ifndef BUILDOPT_HTTP2 + http2_default = FALSE; +#endif + gboolean http2; + if (!ostree_repo_get_remote_boolean_option (self, remote_name, - "http2", TRUE, + "http2", http2_default, &http2, error)) goto out; if (!http2) From 13bcc49603b54f117c44e25dc2b457b9f25d9dc0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 11 Jan 2019 10:46:19 -0500 Subject: [PATCH 29/29] Release 2019.1 Closes: #1800 Approved by: jlebon --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 2fe51589..101a8a17 100644 --- a/configure.ac +++ b/configure.ac @@ -3,11 +3,11 @@ dnl If doing a final release, remember to follow the instructions to dnl update libostree-released.sym from libostree-devel.sym, and update the checksum dnl in test-symbols.sh, and also set is_release_build=yes below. Then make dnl another post-release commit to bump the version, and set is_release_build=no. -m4_define([year_version], [2018]) -m4_define([release_version], [10]) +m4_define([year_version], [2019]) +m4_define([release_version], [1]) m4_define([package_version], [year_version.release_version]) AC_INIT([libostree], [package_version], [walters@verbum.org]) -is_release_build=no +is_release_build=yes AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux])