From 9c195707c629843488d3c712fa8bf17ab052123e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 24 Mar 2016 11:45:10 -0400 Subject: [PATCH 01/69] Add a stub .travis.yml This is at the moment just so that we can use Homu. In the future I'd like to make travis just one of multiple PR testers we use. --- .travis.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..7a80c942 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +# We're just using a "stubbed out" Travis right now so we can +# use Homu to auto-squash +# etc. +# +# In the future we'll hook up better tests. +language: c +dist: trusty +addons: + apt: + packages: + - automake + - autotools-dev +script: + - env NOCONFIGURE=1 ./autogen.sh + +notifications: + # This is Colin's personal Homu instance. We will + # also work on productizing this in Project Atomic. + webhooks: http://escher.verbum.org:54856/travis + email: false + +branches: + only: + - auto + From f84c198006e7ff2d669539b10011a5954bb1e780 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 23 Mar 2016 15:18:27 -0400 Subject: [PATCH 02/69] tests: Add a test-abi This would have caught a potential ABI regression in https://bugzilla.gnome.org/show_bug.cgi?id=764056 (If we ran this test while building `--without-libarchive`) Pull request: #218 Approved by: jlebon --- Makefile-tests.am | 4 ++++ tests/test-abi.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 tests/test-abi.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 9f359ad8..50c714a9 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -23,8 +23,12 @@ include $(top_srcdir)/buildutil/glib-tap.mk # include the builddir in $PATH so we find our just-built ostree # binary. TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ + SRCDIR=$$(cd $(top_srcdir) && pwd) \ + BUILDDIR=$$(cd $(top_builddir) && pwd) \ PATH=$$(cd $(top_builddir) && pwd):$${PATH} +uninstalled_test_scripts = tests/test-abi.sh + test_scripts = \ tests/test-basic.sh \ tests/test-pull-subpath.sh \ diff --git a/tests/test-abi.sh b/tests/test-abi.sh new file mode 100755 index 00000000..a49f8d4b --- /dev/null +++ b/tests/test-abi.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +echo '1..1' + +grep ' ostree_[A-Za-z0-9_]*;' ${SRCDIR}/src/libostree/libostree.sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt +eu-readelf -a ${BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9_.]*,\1,' |sort -u > found-symbols.txt +diff -u expected-symbols.txt found-symbols.txt + +echo 'ok' From ed1e0c6d04244847e489b1b2cb2a3e050fb3a80d Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Mar 2016 11:47:44 +0100 Subject: [PATCH 03/69] pull: Add OSTREE_REPO_PULL_FLAGS_UNTRUSTED flag If this is set we verify all objects we pull, even for local remotes, and we avoid hard-linking into local source repos. https://bugzilla.gnome.org/show_bug.cgi?id=764125 Closes: #221 Approved by: cgwalters --- apidoc/ostree-sections.txt | 1 + src/libostree/libostree.sym | 3 +- src/libostree/ostree-repo-pull.c | 14 ++--- src/libostree/ostree-repo.c | 93 +++++++++++++++++++++++++++----- src/libostree/ostree-repo.h | 12 ++++- 5 files changed, 100 insertions(+), 23 deletions(-) diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 87c138d8..1eef5da7 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -269,6 +269,7 @@ ostree_repo_load_file ostree_repo_load_object_stream ostree_repo_query_object_storage_size ostree_repo_import_object_from +ostree_repo_import_object_from_with_trust ostree_repo_delete_object OstreeRepoCommitFilterResult OstreeRepoCommitFilter diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 7f10b75d..47015e14 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -328,8 +328,7 @@ global: * NOTE NOTE NOTE */ -/* UNCOMMENT WITH NEW SYMBOLS HERE: LIBOSTREE_2016.5 { global: + ostree_repo_import_object_from_with_trust; } LIBOSTREE_2016.4; -*/ diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ecbd7386..26284806 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -93,6 +93,7 @@ typedef struct { gboolean is_mirror; gboolean is_commit_only; + gboolean is_untrusted; char *dir; gboolean commitpartial_exists; @@ -473,9 +474,9 @@ scan_dirtree_object (OtPullData *pull_data, if (!file_is_stored && pull_data->remote_repo_local) { - if (!ostree_repo_import_object_from (pull_data->repo, pull_data->remote_repo_local, - OSTREE_OBJECT_TYPE_FILE, file_checksum, - cancellable, error)) + if (!ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + OSTREE_OBJECT_TYPE_FILE, file_checksum, !pull_data->is_untrusted, + cancellable, error)) goto out; } else if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum)) @@ -1189,9 +1190,9 @@ scan_one_metadata_object_c (OtPullData *pull_data, if (pull_data->remote_repo_local) { if (!is_stored && - !ostree_repo_import_object_from (pull_data->repo, pull_data->remote_repo_local, - objtype, tmp_checksum, - cancellable, error)) + !ostree_repo_import_object_from_with_trust (pull_data->repo, pull_data->remote_repo_local, + objtype, tmp_checksum, !pull_data->is_untrusted, + cancellable, error)) goto out; is_stored = TRUE; is_requested = TRUE; @@ -1931,6 +1932,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; + pull_data->is_untrusted = (flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) > 0; if (error) pull_data->async_error = &pull_data->cached_async_error; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 09791c6a..a5d0bb2d 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3310,24 +3310,37 @@ import_one_object_copy (OstreeRepo *self, OstreeRepo *source, const char *checksum, OstreeObjectType objtype, + gboolean trusted, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; guint64 length; - g_autoptr(GInputStream) object = NULL; + g_autoptr(GInputStream) object_stream = NULL; if (!ostree_repo_load_object_stream (source, objtype, checksum, - &object, &length, + &object_stream, &length, cancellable, error)) goto out; if (objtype == OSTREE_OBJECT_TYPE_FILE) { - if (!ostree_repo_write_content_trusted (self, checksum, - object, length, - cancellable, error)) - goto out; + if (trusted) + { + if (!ostree_repo_write_content_trusted (self, checksum, + object_stream, length, + cancellable, error)) + goto out; + } + else + { + g_autofree guchar *real_csum = NULL; + if (!ostree_repo_write_content (self, checksum, + object_stream, length, + &real_csum, + cancellable, error)) + goto out; + } } else { @@ -3336,10 +3349,29 @@ import_one_object_copy (OstreeRepo *self, if (!copy_detached_metadata (self, source, checksum, cancellable, error)) goto out; } - if (!ostree_repo_write_metadata_stream_trusted (self, objtype, - checksum, object, length, - cancellable, error)) - goto out; + + if (trusted) + { + if (!ostree_repo_write_metadata_stream_trusted (self, objtype, + checksum, object_stream, length, + cancellable, error)) + goto out; + } + else + { + g_autofree guchar *real_csum = NULL; + g_autoptr(GVariant) variant = NULL; + + if (!ostree_repo_load_variant (source, objtype, checksum, + &variant, error)) + goto out; + + if (!ostree_repo_write_metadata (self, objtype, + checksum, variant, + &real_csum, + cancellable, error)) + goto out; + } } ret = TRUE; @@ -3419,11 +3451,43 @@ ostree_repo_import_object_from (OstreeRepo *self, const char *checksum, GCancellable *cancellable, GError **error) +{ + return + ostree_repo_import_object_from_with_trust (self, source, objtype, + checksum, TRUE, cancellable, error); +} + +/** + * ostree_repo_import_object_from_with_trust: + * @self: Destination repo + * @source: Source repo + * @objtype: Object type + * @checksum: checksum + * @trusted: If %TRUE, assume the source repo is valid and trusted + * @cancellable: Cancellable + * @error: Error + * + * Copy object named by @objtype and @checksum into @self from the + * source repository @source. If both repositories are of the same + * type and on the same filesystem, this will simply be a fast Unix + * hard link operation. + * + * Otherwise, a copy will be performed. + */ +gboolean +ostree_repo_import_object_from_with_trust (OstreeRepo *self, + OstreeRepo *source, + OstreeObjectType objtype, + const char *checksum, + gboolean trusted, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; gboolean hardlink_was_supported = FALSE; - - if (self->mode == source->mode) + + if (trusted && /* Don't hardlink into untrusted remotes */ + self->mode == source->mode) { if (!import_one_object_link (self, source, checksum, objtype, &hardlink_was_supported, @@ -3438,10 +3502,10 @@ ostree_repo_import_object_from (OstreeRepo *self, if (!ostree_repo_has_object (self, objtype, checksum, &has_object, cancellable, error)) goto out; - + if (!has_object) { - if (!import_one_object_copy (self, source, checksum, objtype, + if (!import_one_object_copy (self, source, checksum, objtype, trusted, cancellable, error)) goto out; } @@ -3452,6 +3516,7 @@ ostree_repo_import_object_from (OstreeRepo *self, return ret; } + /** * ostree_repo_query_object_storage_size: * @self: Repo diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 07e76aa8..f83aef23 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -423,6 +423,14 @@ gboolean ostree_repo_import_object_from (OstreeRepo *self, const char *checksum, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_import_object_from_with_trust (OstreeRepo *self, + OstreeRepo *source, + OstreeObjectType objtype, + const char *checksum, + gboolean trusted, + GCancellable *cancellable, + GError **error); _OSTREE_PUBLIC gboolean ostree_repo_delete_object (OstreeRepo *self, @@ -891,11 +899,13 @@ gboolean ostree_repo_prune (OstreeRepo *self, * @OSTREE_REPO_PULL_FLAGS_NONE: No special options for pull * @OSTREE_REPO_PULL_FLAGS_MIRROR: Write out refs suitable for mirrors * @OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY: Fetch only the commit metadata + * @OSTREE_REPO_PULL_FLAGS_UNTRUSTED: Don't trust local remote */ typedef enum { OSTREE_REPO_PULL_FLAGS_NONE, OSTREE_REPO_PULL_FLAGS_MIRROR = (1 << 0), - OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY = (1 << 1) + OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY = (1 << 1), + OSTREE_REPO_PULL_FLAGS_UNTRUSTED = (1 << 2) } OstreeRepoPullFlags; _OSTREE_PUBLIC From 456f515522b31bb59f03e2b30a2d86b0faa7d106 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 23 Mar 2016 12:32:03 +0100 Subject: [PATCH 04/69] Add --untrusted option to pull and pull-local https://bugzilla.gnome.org/show_bug.cgi?id=764125 Closes: #221 Approved by: cgwalters --- Makefile-tests.am | 1 + man/ostree-pull-local.xml | 8 ++++ man/ostree-pull.xml | 8 ++++ src/ostree/ot-builtin-pull-local.c | 8 +++- src/ostree/ot-builtin-pull.c | 5 +++ tests/test-pull-untrusted.sh | 63 ++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 1 deletion(-) create mode 100755 tests/test-pull-untrusted.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 50c714a9..ce562265 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -48,6 +48,7 @@ test_scripts = \ tests/test-pull-metalink.sh \ tests/test-pull-summary-sigs.sh \ tests/test-pull-resume.sh \ + tests/test-pull-untrusted.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ tests/test-admin-upgrade-unconfigured.sh \ diff --git a/man/ostree-pull-local.xml b/man/ostree-pull-local.xml index 2ecd12c1..67898743 100644 --- a/man/ostree-pull-local.xml +++ b/man/ostree-pull-local.xml @@ -80,6 +80,14 @@ Boston, MA 02111-1307, USA. Do no invoke fsync(). + + + + + + Do not trust source, verify checksums and don't hardlink into source. + + diff --git a/man/ostree-pull.xml b/man/ostree-pull.xml index c419307e..24ab0b72 100644 --- a/man/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -73,6 +73,14 @@ Boston, MA 02111-1307, USA. + + + + + Do not trust local sources, verify checksums and don't hardlink into source. + + + diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index ed87d806..f3ca184a 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -32,11 +32,13 @@ static char *opt_remote; static gboolean opt_disable_fsync; +static gboolean opt_untrusted; static int opt_depth = 0; static GOptionEntry options[] = { { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, + { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust source", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -54,6 +56,7 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr glnx_unref_object OstreeAsyncProgress *progress = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; g_autoptr(GHashTable) source_objects = NULL; + OstreeRepoPullFlags pullflags = 0; context = g_option_context_new ("SRC_REPO [REFS...] - Copy data from SRC_REPO"); @@ -83,6 +86,9 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr src_repo_uri = g_strconcat ("file://", cwd, "/", src_repo_arg, NULL); } + if (opt_untrusted) + pullflags |= OSTREE_REPO_PULL_FLAGS_UNTRUSTED; + if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); @@ -133,7 +139,7 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); g_variant_builder_add (&builder, "{s@v}", "flags", - g_variant_new_variant (g_variant_new_int32 (OSTREE_REPO_PULL_FLAGS_NONE))); + g_variant_new_variant (g_variant_new_int32 (pullflags))); g_variant_builder_add (&builder, "{s@v}", "refs", g_variant_new_variant (g_variant_new_strv ((const char *const*) refs_to_fetch->pdata, -1))); if (opt_remote) diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 7c91890f..8bef63a3 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -33,6 +33,7 @@ static gboolean opt_commit_only; static gboolean opt_dry_run; static gboolean opt_disable_static_deltas; static gboolean opt_require_static_deltas; +static gboolean opt_untrusted; static char* opt_subpath; static int opt_depth = 0; @@ -43,6 +44,7 @@ static GOptionEntry options[] = { { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, + { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust (local) sources", NULL }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } @@ -134,6 +136,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_commit_only) pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY; + if (opt_untrusted) + pullflags |= OSTREE_REPO_PULL_FLAGS_UNTRUSTED; + if (opt_dry_run && !opt_require_static_deltas) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, diff --git a/tests/test-pull-untrusted.sh b/tests/test-pull-untrusted.sh new file mode 100755 index 00000000..95f7ab93 --- /dev/null +++ b/tests/test-pull-untrusted.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Copyright (C) 2014 Alexander Larsson +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..3' + +setup_test_repository "bare" + +cd ${test_tmpdir} +mkdir repo2 +${CMD_PREFIX} ostree --repo=repo2 init --mode="bare" + +${CMD_PREFIX} ostree --repo=repo2 --untrusted pull-local repo + +find repo2 -type f -links +1 | while read line; do + assert_not_reached "pull-local created hardlinks" +done +echo "ok pull-local --untrusted didn't hardlink" + +# Corrupt repo +for i in ${test_tmpdir}/repo/objects/*/*.file; do + echo "corrupting $i" + echo "broke" >> $i + break; +done + +rm -rf repo2 +mkdir repo2 +${CMD_PREFIX} ostree --repo=repo2 init --mode="bare" +if ${CMD_PREFIX} ostree --repo=repo2 pull-local repo; then + echo "ok trusted pull with corruption succeeded" +else + assert_not_reached "corrupted trusted pull unexpectedly succeeded!" +fi + +rm -rf repo2 +mkdir repo2 +${CMD_PREFIX} ostree --repo=repo2 init --mode="bare" +if ${CMD_PREFIX} ostree --repo=repo2 pull-local --untrusted repo; then + assert_not_reached "corrupted untrusted pull unexpectedly failed!" +else + echo "ok untrusted pull with corruption failed" +fi From b7a04d51f8bc95c6168de155bf625e1ff0ea85ee Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 23 Mar 2016 15:54:49 -0400 Subject: [PATCH 05/69] OstreeSePolicy: add ostree_sepolicy_get_csum() This can be used as a fingerprint to determine whether two OstreeSePolicy objects are equivalent. Also add documentation for ostree_sepolicy_get_name(). Closes: #219 Approved by: cgwalters --- src/libostree/libostree.sym | 1 + src/libostree/ostree-sepolicy.c | 117 ++++++++++++++++++++++++++++++ src/libostree/ostree-sepolicy.h | 3 + src/libotutil/ot-checksum-utils.c | 12 +-- src/libotutil/ot-checksum-utils.h | 9 ++- 5 files changed, 132 insertions(+), 10 deletions(-) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 47015e14..ca3c6fd0 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -331,4 +331,5 @@ global: LIBOSTREE_2016.5 { global: ostree_repo_import_object_from_with_trust; + ostree_sepolicy_get_csum; } LIBOSTREE_2016.4; diff --git a/src/libostree/ostree-sepolicy.c b/src/libostree/ostree-sepolicy.c index b8e3572d..3b1a391b 100644 --- a/src/libostree/ostree-sepolicy.c +++ b/src/libostree/ostree-sepolicy.c @@ -50,6 +50,7 @@ struct OstreeSePolicy { GFile *selinux_policy_root; struct selabel_handle *selinux_hnd; char *selinux_policy_name; + char *selinux_policy_csum; #endif }; @@ -77,6 +78,7 @@ ostree_sepolicy_finalize (GObject *object) #ifdef HAVE_SELINUX g_clear_object (&self->selinux_policy_root); g_clear_pointer (&self->selinux_policy_name, g_free); + g_clear_pointer (&self->selinux_policy_csum, g_free); if (self->selinux_hnd) { selabel_close (self->selinux_hnd); @@ -155,6 +157,93 @@ ostree_sepolicy_class_init (OstreeSePolicyClass *klass) G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); } +#ifdef HAVE_SELINUX + +/* Find the latest policy file in our root and return its checksum. */ +static gboolean +get_policy_checksum (char **out_csum, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + + const char *binary_policy_path = selinux_binary_policy_path (); + const char *binfile_prefix = glnx_basename (binary_policy_path); + g_autofree char *bindir_path = g_path_get_dirname (binary_policy_path); + + glnx_fd_close int bindir_dfd = -1; + + g_autofree char *best_policy = NULL; + int best_version = 0; + + g_auto(GLnxDirFdIterator) dfd_iter = { 0,}; + + if (!glnx_opendirat (AT_FDCWD, bindir_path, TRUE, &bindir_dfd, error)) + goto out; + + if (!glnx_dirfd_iterator_init_at (bindir_dfd, ".", FALSE, &dfd_iter, error)) + goto out; + + while (TRUE) + { + struct dirent *dent = NULL; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, + cancellable, error)) + goto out; + + if (dent == NULL) + break; + + if (dent->d_type == DT_REG) + { + /* We could probably save a few hundred nanoseconds if we accept that + * the prefix will always be "policy" and hardcode that in a static + * compile-once GRegex... But picture how exciting it'd be if it *did* + * somehow change; there would be cheers & slow-mo high-fives at the + * sight of our code not breaking. Is that hope not worth a fraction + * of a millisecond? I believe it is... or maybe I'm just lazy. */ + g_autofree char *regex = g_strdup_printf ("^\\Q%s\\E\\.[0-9]+$", + binfile_prefix); + + /* we could use match groups to extract the version, but mehhh, we + * already have the prefix on hand */ + if (g_regex_match_simple (regex, dent->d_name, 0, 0)) + { + int version = /* do +1 for the period */ + (int)g_ascii_strtoll (dent->d_name + strlen (binfile_prefix)+1, + NULL, 10); + g_assert (version > 0); + + if (version > best_version) + { + best_version = version; + g_free (best_policy); + best_policy = g_strdup (dent->d_name); + } + } + } + } + + if (!best_policy) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Could not find binary policy file"); + goto out; + } + + *out_csum = ot_checksum_file_at (bindir_dfd, best_policy, G_CHECKSUM_SHA256, + cancellable, error); + if (*out_csum == NULL) + goto out; + + ret = TRUE; +out: + return ret; +} + +#endif + static gboolean initable_init (GInitable *initable, GCancellable *cancellable, @@ -257,6 +346,12 @@ initable_init (GInitable *initable, freecon (con); } + if (!get_policy_checksum (&self->selinux_policy_csum, cancellable, error)) + { + g_prefix_error (error, "While calculating SELinux checksum: "); + goto out; + } + self->selinux_policy_name = g_strdup (policytype); self->selinux_policy_root = g_object_ref (etc_selinux_dir); } @@ -306,6 +401,12 @@ ostree_sepolicy_get_path (OstreeSePolicy *self) return self->path; } +/** + * ostree_sepolicy_get_name: + * @self: + * + * Returns: (transfer none): Type of current policy + */ const char * ostree_sepolicy_get_name (OstreeSePolicy *self) { @@ -316,6 +417,22 @@ ostree_sepolicy_get_name (OstreeSePolicy *self) #endif } +/** + * ostree_sepolicy_get_csum: + * @self: + * + * Returns: (transfer none): Checksum of current policy + */ +const char * +ostree_sepolicy_get_csum (OstreeSePolicy *self) +{ +#ifdef HAVE_SELINUX + return self->selinux_policy_csum; +#else + return NULL; +#endif +} + /** * ostree_sepolicy_get_label: * @self: Self diff --git a/src/libostree/ostree-sepolicy.h b/src/libostree/ostree-sepolicy.h index 83e3b379..d204953e 100644 --- a/src/libostree/ostree-sepolicy.h +++ b/src/libostree/ostree-sepolicy.h @@ -44,6 +44,9 @@ GFile * ostree_sepolicy_get_path (OstreeSePolicy *self); _OSTREE_PUBLIC const char *ostree_sepolicy_get_name (OstreeSePolicy *self); +_OSTREE_PUBLIC +const char *ostree_sepolicy_get_csum (OstreeSePolicy *self); + _OSTREE_PUBLIC gboolean ostree_sepolicy_get_label (OstreeSePolicy *self, const char *relpath, diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c index b2aaf329..8d30bdc3 100644 --- a/src/libotutil/ot-checksum-utils.c +++ b/src/libotutil/ot-checksum-utils.c @@ -140,17 +140,17 @@ ot_gio_checksum_stream (GInputStream *in, } char * -ot_checksum_file (GFile *file, - GChecksumType checksum_type, - GCancellable *cancellable, - GError **error) +ot_checksum_file_at (int dfd, + const char *path, + GChecksumType checksum_type, + GCancellable *cancellable, + GError **error) { GChecksum *checksum = NULL; char *ret = NULL; g_autoptr(GInputStream) in = NULL; - in = (GInputStream*)g_file_read (file, cancellable, error); - if (!in) + if (!ot_openat_read_stream (dfd, path, TRUE, &in, cancellable, error)) goto out; checksum = g_checksum_new (checksum_type); diff --git a/src/libotutil/ot-checksum-utils.h b/src/libotutil/ot-checksum-utils.h index eb8bbc04..8b3a394e 100644 --- a/src/libotutil/ot-checksum-utils.h +++ b/src/libotutil/ot-checksum-utils.h @@ -53,10 +53,11 @@ gboolean ot_gio_checksum_stream (GInputStream *in, GCancellable *cancellable, GError **error); -char * ot_checksum_file (GFile *file, - GChecksumType checksum_type, - GCancellable *cancellable, - GError **error); +char * ot_checksum_file_at (int dfd, + const char *path, + GChecksumType checksum_type, + GCancellable *cancellable, + GError **error); void ot_gio_checksum_stream_async (GInputStream *in, int io_priority, From 31240982e7aec400a39ee1e71ca80d1ca67b0d4a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Thu, 17 Mar 2016 11:39:39 -0700 Subject: [PATCH 06/69] core: Add verbose messages for pruning When prune fails, it can be really difficult to figure out why. This at least lets you know which objects are being considered. https://bugzilla.gnome.org/show_bug.cgi?id=764006 Closes: #224 Approved by: cgwalters --- src/libostree/ostree-repo-prune.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index eca2cff6..9aed0ac3 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -74,6 +74,8 @@ maybe_prune_loose_object (OtPruneData *data, if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL)) { + g_debug ("Pruning unneeded object %s.%s", checksum, + ostree_object_type_to_string (objtype)); if (!(flags & OSTREE_REPO_PRUNE_FLAGS_NO_PRUNE)) { guint64 storage_size = 0; @@ -101,6 +103,8 @@ maybe_prune_loose_object (OtPruneData *data, } else { + g_debug ("Keeping needed object %s.%s", checksum, + ostree_object_type_to_string (objtype)); if (OSTREE_OBJECT_TYPE_IS_META (objtype)) data->n_reachable_meta++; else @@ -234,6 +238,7 @@ ostree_repo_prune_static_deltas (OstreeRepo *self, const char *commit, continue; } + g_debug ("Trying to prune static delta %s", deltaname); deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); if (!glnx_shutil_rm_rf_at (self->repo_dir_fd, deltadir, @@ -310,6 +315,7 @@ ostree_repo_prune (OstreeRepo *self, error)) goto out; + g_debug ("Finding objects to keep for commit %s", checksum); if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable, cancellable, &local_error)) { @@ -352,6 +358,7 @@ ostree_repo_prune (OstreeRepo *self, error)) goto out; + g_debug ("Finding objects to keep for commit %s", checksum); if (!ostree_repo_traverse_commit_union (self, checksum, depth, data.reachable, cancellable, &local_error)) { From 750e2cbf33503ef0e08da93bba110ab3a2f46e8a Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Mon, 21 Mar 2016 15:55:19 -0700 Subject: [PATCH 07/69] core: Add debug messages for traversing If you have a repo where a needed object has been inadvertantly removed, all you'll get is a "No such metadata object" error with no clue about where it was referenced from. Add some debug messages to provide clues about which objects are being traversed and found. https://bugzilla.gnome.org/show_bug.cgi?id=764006 Closes: #224 Approved by: cgwalters --- src/libostree/ostree-repo-traverse.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libostree/ostree-repo-traverse.c b/src/libostree/ostree-repo-traverse.c index 97bd1023..bb437c38 100644 --- a/src/libostree/ostree-repo-traverse.c +++ b/src/libostree/ostree-repo-traverse.c @@ -328,6 +328,7 @@ traverse_iter (OstreeRepo *repo, ostree_repo_commit_traverse_iter_get_file (iter, &name, &checksum); + g_debug ("Found file object %s", checksum); key = ostree_object_name_serialize (checksum, OSTREE_OBJECT_TYPE_FILE); g_hash_table_replace (inout_reachable, key, key); key = NULL; @@ -341,6 +342,8 @@ traverse_iter (OstreeRepo *repo, ostree_repo_commit_traverse_iter_get_dir (iter, &name, &content_checksum, &meta_checksum); + g_debug ("Found dirtree object %s", content_checksum); + g_debug ("Found dirmeta object %s", meta_checksum); key = ostree_object_name_serialize (meta_checksum, OSTREE_OBJECT_TYPE_DIR_META); g_hash_table_replace (inout_reachable, key, key); key = NULL; @@ -381,6 +384,7 @@ traverse_dirtree (OstreeRepo *repo, &dirtree, error)) goto out; + g_debug ("Traversing dirtree %s", checksum); if (!ostree_repo_commit_traverse_iter_init_dirtree (&iter, repo, dirtree, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, error)) @@ -444,6 +448,7 @@ ostree_repo_traverse_commit_union (OstreeRepo *repo, g_hash_table_add (inout_reachable, key); key = NULL; + g_debug ("Traversing commit %s", commit_checksum); if (!ostree_repo_commit_traverse_iter_init_commit (&iter, repo, commit, OSTREE_REPO_COMMIT_TRAVERSE_FLAG_NONE, error)) From e9c58fe706839ee13d2b4f0346271a121998070f Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Sat, 26 Mar 2016 08:28:20 -0700 Subject: [PATCH 08/69] build: Set G_LOG_DOMAIN to OSTree This will allow ostree programs to filter log messages specifically for OSTree instead of using the NULL domain for ostree debugging. https://bugzilla.gnome.org/show_bug.cgi?id=764237 Closes: #225 Approved by: cgwalters --- Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.am b/Makefile.am index a485cc75..e25842d9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -24,6 +24,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DLOCALEDIR=\"$(datadir)/locale\" -DSYSCONFDIR=\"$(sysconfdir)\" \ -DSHORTENED_SYSCONFDIR=\"$(shortened_sysconfdir)\" \ -DOSTREE_FEATURES='"$(OSTREE_FEATURES)"' \ + -DG_LOG_DOMAIN=\"OSTree\" \ -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 AM_CFLAGS += $(WARN_CFLAGS) From b1e1e5166018aac690af841232a1a2c9c1d835c4 Mon Sep 17 00:00:00 2001 From: Dan Nicholson Date: Sat, 26 Mar 2016 08:35:30 -0700 Subject: [PATCH 09/69] main: Set log handler for OSTree domain Now that OSTree is used as G_LOG_DOMAIN, set the main handler to match so the appropriate messages are filtered. It would probably be more appropriate to spell out "OSTree" in the code, but since G_LOG_DOMAIN is being defined globally in the project, might as well reuse it here. https://bugzilla.gnome.org/show_bug.cgi?id=764237 Closes: #225 Approved by: cgwalters --- src/ostree/ot-main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-main.c b/src/ostree/ot-main.c index 51282f92..93f841dc 100644 --- a/src/ostree/ot-main.c +++ b/src/ostree/ot-main.c @@ -132,7 +132,7 @@ ostree_run (int argc, /* avoid gvfs (http://bugzilla.gnome.org/show_bug.cgi?id=526454) */ g_setenv ("GIO_USE_VFS", "local", TRUE); - g_log_set_handler (NULL, G_LOG_LEVEL_MESSAGE, message_handler, NULL); + g_log_set_handler (G_LOG_DOMAIN, G_LOG_LEVEL_MESSAGE, message_handler, NULL); /* * Parse the global options. We rearrange the options as @@ -255,7 +255,7 @@ ostree_option_context_parse (GOptionContext *context, } if (opt_verbose) - g_log_set_handler (NULL, G_LOG_LEVEL_DEBUG, message_handler, NULL); + g_log_set_handler (G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, message_handler, NULL); if (opt_repo == NULL && !(flags & OSTREE_BUILTIN_FLAG_NO_REPO)) { From 45a6c109d959e9ba92e39397a2cb9da8c1cf67af Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 28 Mar 2016 12:25:17 +0100 Subject: [PATCH 10/69] packaging: fix bashism in dist-snapshot target On Debian and its derivatives, /bin/sh is a lightweight POSIX shell (currently dash) which does not support the bash {foo,bar} syntax. Signed-off-by: Simon McVittie Closes: #226 Approved by: cgwalters --- packaging/Makefile.dist-packaging | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile.dist-packaging b/packaging/Makefile.dist-packaging index b55211d6..096aeaba 100644 --- a/packaging/Makefile.dist-packaging +++ b/packaging/Makefile.dist-packaging @@ -22,7 +22,7 @@ dist-snapshot: tar -A -f $${TARFILE_TMP} submodule.tar; \ rm submodule.tar; \ done; \ - mv $(PKG_VER).tar{.tmp,}; \ + mv $(PKG_VER).tar.tmp $(PKG_VER).tar; \ rm -f $(PKG_VER).tar.xz; \ xz $(PKG_VER).tar From a50df5daf75cd03dae506eb8e5d2e57f71bb3de3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 25 Mar 2016 10:35:25 -0400 Subject: [PATCH 11/69] docs: Add a section on repository management Just keeping my promise to write more documentation. There could be a lot more to write here, but I'm trying to get a start done. Closes: #222 Approved by: jlebon --- docs/manual/buildsystem-and-repos.md | 5 + docs/manual/repository-management.md | 214 +++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 220 insertions(+) create mode 100644 docs/manual/repository-management.md diff --git a/docs/manual/buildsystem-and-repos.md b/docs/manual/buildsystem-and-repos.md index d418cb0e..2da3d8d7 100644 --- a/docs/manual/buildsystem-and-repos.md +++ b/docs/manual/buildsystem-and-repos.md @@ -178,3 +178,8 @@ commit. ``` ostree --repo=repo static-delta generate exampleos/x86_64/standard ``` + +## More sophisticated repository management + +Next, see [Repository Management](repository-management.md) for the +next steps in managing content in OSTree repositories. diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md new file mode 100644 index 00000000..df92f6b9 --- /dev/null +++ b/docs/manual/repository-management.md @@ -0,0 +1,214 @@ +# Managing content in OSTree repositories + +Once you have a build system going, if you actually want client +systems to retrieve the content, you will quickly feel a need for +"repository management". + +OSTree itself does not currently come with tools to do this. One +reason is that how content is delivered and managed has concerns very +specific to the organization. For example, some operating system +content vendors may want integration with a specific errata +notification system. + +In this section, we will describe some high level ideas and methods +for managing content in OSTree repositories, mostly independent of any +particular model or tool. That said, a goal is to include at least +some sample scripts and workflows upstream in a potential new +"contrib" git repository. + +One example of software which can assist in managing OSTree +repositories today is the [Pulp Project](http://www.pulpproject.org/), +which has a +[Pulp OSTree plugin](https://pulp-ostree.readthedocs.org/en/latest/). + +## Separate development vs release repositories + +By default, OSTree accumulates server side history. This is actually +optional in that your build system can (using the API) write a commit +with no parent. But first, we'll investigate the ramifications of +server side history. + +Many content vendors will want to separate their internal development +with what is made public to the world. Therefore, you will want (at +least) two OSTree repositories, we'll call them "dev" and "prod". + +To phrase this another way, let's say you have a continuous delivery +system which is building from git and committing into your "dev" +OSTree repository. This might happen tens to hundreds of times per +day. That's a substantial amount of history over time, and it's +unlikely most of your content consumers (i.e. not developers/testers) +will be interested in all of it. + +The original vision of OSTree was to fulfill this "dev" role, and in +particular the "archive-z2" format was designed for it. + +Then, what you'll want to do is promote content from "dev" to "prod". +We'll discuss this later, but first, let's talk about promotion +*inside* our "dev" repository. + +## Promoting content along OSTree branches - "buildmaster", "smoketested" + +Besides multiple repositories, OSTree also supports multiple branches +inside one repository, equivalent to git's branches. We saw in an +earlier section an example branch name like +`exampleos/x86_64/standard`. Choosing the branch name for your "prod" +repository is absolutely critical as client systems will reference it. +It becomes an important part of your face to the world, in the same +way the "master" branch in a git repository is. + +But with your "dev" repository internally, it can be very useful to +use OSTree's branching concepts to represent different stages in a +software delivery pipeline. + +Deriving from `exampleos/x86_64/standard`, let's say our "dev" +repository contains `exampleos/x86_64/buildmaster/standard`. We choose the +term "buildmaster" to represent something that came straight from git +master. It may not be tested very much. + +Our next step should be to hook up a testing system (Jenkins, +Buildbot, etc.) to this. When a build (commit) passes some tests, we +want to "promote" that commit. Let's create a new branch called +`smoketested` to say that some basic sanity checks pass on the +complete system. This might be where human testers get involved, for +example. + +The build system can "promote" the `buildmaster` commit that passed +testing like this: + +``` +ostree commit -b exampleos/x86_64/smoketested/standard -s 'Passed tests' --tree=ref=aec070645fe53... +``` + +Here we're generating a new commit object (perhaps include in the commit +log links to build logs, etc.), but we're reusing the *content* from the `buildmaster` +commit `aec070645fe53` that passed the smoketests. + +We can easily generalize this model to have an arbitrary number of +stages like `exampleos/x86_64/stage-1-pass/standard`, +`exampleos/x86_64/stage-2-pass/standard`, etc. depending on business +requirements and logic. + +In this suggested model, the "stages" are increasingly expensive. The +logic is that we don't want to spend substantial time on e.g. network +performance tests if something basic like a systemd unit file fails on +bootup. + + +## Promoting content between OSTree repositories + +Now, we have our internal continuous delivery stream flowing, it's +being tested and works. We want to periodically take the latest +commit on `exampleos/x86_64/stage-3-pass/standard` and expose it in +our "prod" repository as `exampleos/x86_64/standard`, with a much +smaller history. + +We'll have other business requirements such as writing release notes +(and potentially putting them in the OSTree commit message), etc. + +In [Build Systems](buildsystem-and-repos.md) we saw how the +`pull-local` command can be used to migrate content from the "build" +repository (in `bare-user` mode) into an `archive-z2` repository for +serving to client systems. + +Following this section, we now have three repositories, let's call +them `repo-build`, `repo-dev`, and `repo-prod`. We've been pulling +content from `repo-build` into `repo-dev` (which involves gzip +compression among other things since it is a format change). + +When using `pull-local` to migrate content between two `archive-z2` +repositories, the binary content is taken unmodified. Let's go ahead +and generate a new commit in our prod repository: + +``` +checksum=$(ostree --repo=repo-dev rev-parse exampleos/x86_64/stage-3-pass/standard`) +ostree --repo=repo-prod pull-local repo-dev ${checksum} +ostree --repo=repo-prod commit -b exampleos/x86_64/standard \ + -s 'Release 1.2.3' --add-metadata-string=ostree.version=1.2.3 \ + --tree=ref=${checksum} +``` + +There are a few things going on here. First, we found the latest +commit checksum for the "stage-3 dev", and told `pull-local` to copy +it, without using the branch name. We do this because we don't want +to expose the `exampleos/x86_64/stage-3-pass/standard` branch name in +our "prod" repository. + +Next, we generate a new commit in prod that's referencing the exact +binary content in dev. If the "dev" and "prod" repositories are on +the same Unix filesystem, (like git) OSTree will make use of hard +links to avoid copying any content at all - making the process very +fast. + +Another interesting thing to notice here is that we're adding an +`ostree.version` metadata string to the commit. This is an optional +piece of metadata, but we are encouraging its use in the OSTree +ecosystem of tools. Commands like `ostree admin status` show it by +default. + +## Derived data - static deltas and the summary file + +As discussed in [Formats](formats.md), the `archive-z2` repository we +use for "prod" requires one HTTP fetch per client request by default. +If we're only performing a release e.g. once a week, it's appropriate +to use "static deltas" to speed up client updates. + +So once we've used the above command to pull content from `repo-dev` +into `repo-prod`, let's generate a delta against the previous commit: + +``` +ostree --repo=repo-prod static-delta generate exampleos/x86_64/standard +``` + +We may also want to support client systems upgrading from *two* +commits previous. + +``` +ostree --repo=repo-prod static-delta generate --from=exampleos/x86_64/standard^^ --to=exampleos/x86_64/standard +``` + +Generating a full permutation of deltas across all prior versions can +get expensive, and there is some support in the OSTree core for static +deltas which "recurse" to a parent. This can help create a model +where clients download a chain of deltas. Support for this is not +fully implemented yet however. + +Regardless of whether or not you choose to generate static deltas, +you should update the summary file: + +``` +ostree --repo=repo-prod summary -u +``` + +(Remember, the `summary` command can not be run concurrently, so this + should be triggered serially by other jobs). + +There is some more information on the design of the summary file in +[Repo](repo.md). + +## Pruning our build and dev repositories + +First, the OSTree author believes you should *not* use OSTree as a +"primary content store". The binaries in an OSTree repository should +be derived from a git repository. Your build system should record +proper metadata such as the configuration options used to generate the +build, and you should be able to rebuild it if necessary. Art assets +should be stored in a system that's designed for that +(e.g. [Git LFS](https://git-lfs.github.com/)). + +Another way to say this is that five years down the line, we are +unlikely to care about retaining the exact binaries from an OS build +on Wednesday afternoon three years ago. + +We want to save space and prune our "dev" repository. + +``` +ostree --repo=repo-dev prune --refs-only --keep-younger-than="6 months ago" +``` + +That will truncate the history older than 6 months. Deleted commits +will have "tombstone markers" added so that you know they were +explicitly deleted, but all content in them (that is not referenced by +a still retained commit) will be garbage collected. + + + diff --git a/mkdocs.yml b/mkdocs.yml index e512ea62..3b882e1c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,4 +10,5 @@ pages: - Adapting Existing Systems: 'manual/adapting-existing.md' - Formats: 'manual/formats.md' - Build Systems and Repos: 'manual/buildsystem-and-repos.md' + - Repository Management: 'manual/repository-management.md' - Related Projects: 'manual/related-projects.md' From c6b4ecd474257fa192743096a8dc94f6ae1e7be7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 25 Mar 2016 11:03:32 -0400 Subject: [PATCH 12/69] commit: Support generating commits with no parent, or a custom one When I'm doing local development builds, it's quite common for me not to want to accumulate history. There are also use cases for this on build servers as well. In particular, using this, one could write a build system that didn't necessarily need to have access to (a copy of) the OSTree repository. Instead, the build system would determine the last commit ID on the branch, and pass that to a worker node, then sync the generated content back. The API supported generating custom commits that don't necessarily reference the previous commit on the same branch, let's just expose this in the command line for convenience. I plan to also support this rpm-ostree. Closes: #223 Approved by: jlebon --- src/ostree/ot-builtin-commit.c | 20 ++++++++++++++++++-- tests/basic-test.sh | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 72b83bf7..517ea70e 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -32,6 +32,7 @@ static char *opt_subject; static char *opt_body; +static char *opt_parent; static char *opt_branch; static char *opt_statoverride_file; static char **opt_metadata_strings; @@ -67,6 +68,7 @@ parse_fsync_cb (const char *option_name, } static GOptionEntry options[] = { + { "parent", 0, 0, G_OPTION_ARG_STRING, &opt_parent, "Parent ref, or \"none\"", "REF" }, { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject", "SUBJECT" }, { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "BODY" }, { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" }, @@ -361,8 +363,22 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError modifier = ostree_repo_commit_modifier_new (flags, commit_filter, mode_adds, NULL); } - if (!ostree_repo_resolve_rev (repo, opt_branch, TRUE, &parent, error)) - goto out; + if (opt_parent) + { + if (g_str_equal (opt_parent, "none")) + parent = NULL; + else + { + if (!ostree_validate_checksum_string (opt_parent, error)) + goto out; + parent = g_strdup (opt_parent); + } + } + else + { + if (!ostree_repo_resolve_rev (repo, opt_branch, TRUE, &parent, error)) + goto out; + } if (!opt_subject && !opt_body) { diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 8acec1c2..4b46fc10 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..50" +echo "1..52" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -96,6 +96,23 @@ assert_file_has_content yet/another/tree/green 'leaf' assert_file_has_content four '4' echo "ok cwd contents" +cd ${test_tmpdir} +$OSTREE commit -b test2-no-parent -s '' $test_tmpdir/checkout-test2-4 +assert_streq $($OSTREE log test2-no-parent |grep '^commit' | wc -l) "1" +$OSTREE commit -b test2-no-parent -s '' --parent=none $test_tmpdir/checkout-test2-4 +assert_streq $($OSTREE log test2-no-parent |grep '^commit' | wc -l) "1" +echo "ok commit no parent" + +cd ${test_tmpdir} +$OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 +$OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 +$OSTREE commit -b test2-custom-parent -s '' $test_tmpdir/checkout-test2-4 +assert_streq $($OSTREE log test2-custom-parent |grep '^commit' | wc -l) "3" +prevparent=$($OSTREE rev-parse test2-custom-parent^) +$OSTREE commit -b test2-custom-parent -s '' --parent=${prevparent} $test_tmpdir/checkout-test2-4 +assert_streq $($OSTREE log test2-custom-parent |grep '^commit' | wc -l) "3" +echo "ok commit custom parent" + cd ${test_tmpdir} $OSTREE diff test2^ test2 > diff-test2 assert_file_has_content diff-test2 'D */a/5' From 23d26d5f65a79d77afba515afd45a46ad3a80209 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 29 Mar 2016 10:53:50 -0400 Subject: [PATCH 13/69] commit: Support writing orphans The API supports this, and it's not hard for us to do in the command line as well. One possible use case is separating "content generation" in a separate server. Related: https://github.com/ostreedev/ostree/pull/223 Closes: #227 Approved by: jlebon --- src/ostree/ot-builtin-commit.c | 15 ++++++++++----- tests/basic-test.sh | 12 +++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 517ea70e..2c74d4d7 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -33,6 +33,7 @@ static char *opt_subject; static char *opt_body; static char *opt_parent; +static gboolean opt_orphan; static char *opt_branch; static char *opt_statoverride_file; static char **opt_metadata_strings; @@ -72,6 +73,7 @@ static GOptionEntry options[] = { { "subject", 's', 0, G_OPTION_ARG_STRING, &opt_subject, "One line subject", "SUBJECT" }, { "body", 'm', 0, G_OPTION_ARG_STRING, &opt_body, "Full description", "BODY" }, { "branch", 'b', 0, G_OPTION_ARG_STRING, &opt_branch, "Branch", "BRANCH" }, + { "orphan", 0, 0, G_OPTION_ARG_NONE, &opt_orphan, "Create a commit without writing a ref", NULL }, { "tree", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_trees, "Overlay the given argument as a tree", "dir=PATH or tar=TARFILE or ref=COMMIT" }, { "add-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_metadata_strings, "Add a key/value pair to metadata", "KEY=VALUE" }, { "add-detached-metadata-string", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_detached_metadata_strings, "Add a key/value pair to detached metadata", "KEY=VALUE" }, @@ -340,10 +342,10 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError goto out; } - if (!opt_branch) + if (!(opt_branch || opt_orphan)) { g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "A branch must be specified with --branch"); + "A branch must be specified with --branch, or use --orphan"); goto out; } @@ -374,13 +376,13 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError parent = g_strdup (opt_parent); } } - else + else if (!opt_orphan) { if (!ostree_repo_resolve_rev (repo, opt_branch, TRUE, &parent, error)) goto out; } - if (!opt_subject && !opt_body) + if (!opt_subject && !opt_body && !opt_orphan) { if (!commit_editor (repo, opt_branch, &opt_subject, &opt_body, cancellable, error)) goto out; @@ -563,7 +565,10 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError } } - ostree_repo_transaction_set_ref (repo, NULL, opt_branch, commit_checksum); + if (opt_branch) + ostree_repo_transaction_set_ref (repo, NULL, opt_branch, commit_checksum); + else + g_assert (opt_orphan); if (!ostree_repo_commit_transaction (repo, &stats, cancellable, error)) goto out; diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 4b46fc10..1ae7e7a4 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..52" +echo "1..53" $OSTREE checkout test2 checkout-test2 echo "ok checkout" @@ -113,6 +113,16 @@ $OSTREE commit -b test2-custom-parent -s '' --parent=${prevparent} $test_tmpdir/ assert_streq $($OSTREE log test2-custom-parent |grep '^commit' | wc -l) "3" echo "ok commit custom parent" +cd ${test_tmpdir} +orphaned_rev=$($OSTREE commit --orphan -s '' $test_tmpdir/checkout-test2-4) +$OSTREE ls ${orphaned_rev} >/dev/null +$OSTREE prune --refs-only +if $OSTREE ls ${orphaned_rev} 2>err.txt; then + assert_not_reached "Found orphaned commit" +fi +assert_file_has_content err.txt "No such metadata object" +echo "ok commit orphaned" + cd ${test_tmpdir} $OSTREE diff test2^ test2 > diff-test2 assert_file_has_content diff-test2 'D */a/5' From 9260d3dba1dc1d22a612161543e46ef22dce18fd Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 29 Mar 2016 23:27:19 -0400 Subject: [PATCH 14/69] commit: support editor for orphan commits This is a follow up to #227 to allow ostree to open the editor also for orphan commits when no subject or body is given on the cmdline. Closes: #229 Approved by: cgwalters --- src/ostree/ot-builtin-commit.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 2c74d4d7..1d96855d 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -194,9 +194,8 @@ commit_editor (OstreeRepo *repo, input = g_strdup_printf ("\n" "# Please enter the commit message for your changes. The first line will\n" "# become the subject, and the remainder the body. Lines starting\n" - "# with '#' will be ignored, and an empty message aborts the commit.\n" - "#\n" - "# Branch: %s\n", branch); + "# with '#' will be ignored, and an empty message aborts the commit." + "%s%s\n", branch ? "\n#\n# Branch: " : "", branch ?: ""); output = ot_editor_prompt (repo, input, cancellable, error); if (output == NULL) @@ -382,7 +381,7 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError goto out; } - if (!opt_subject && !opt_body && !opt_orphan) + if (!opt_subject && !opt_body) { if (!commit_editor (repo, opt_branch, &opt_subject, &opt_body, cancellable, error)) goto out; From 39777ded54efa94dd78896c50b212ffcbadad7da Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 30 Mar 2016 09:19:38 -0400 Subject: [PATCH 15/69] docs/CONTRIBUTING.md: Update for github move, Homu etc. Closes: #230 Approved by: jlebon --- docs/CONTRIBUTING.md | 47 ++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 397ffeb0..a26f3975 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,29 +1,46 @@ Submitting patches ------------------ -You can: +A majority of current maintainers prefer the Github pull request +model, and this motivated moving the primary git repository to +. + +However, we do not use the "Merge pull request" button, because we do +not like merge commits for one-patch pull requests, among other +reasons. See [this issue](https://github.com/isaacs/github/issues/2) +for more information. Instead, we use an instance of +[Homu](https://github.com/servo/homu), currently known as +`cgwalters-bot`. + +As a review proceeeds, the preferred method is to push `fixup!` +commits via `git commit --fixup`. Homu knows how to use +`--autosquash` when performing the final merge. See the +[Git documentation](https://git-scm.com/docs/git-rebase]) for more +information. + +Alternative methods if you don't like Github (also fully supported): 1. Send mail to , with the patch attached - 1. Submit a pull request against 1. Attach them to -Please look at `git log` and match the commit log style. +It is likely however once a patch is ready to apply a maintainer +will push it to a github PR, and merge via Homu. + +Commit message style +-------------------- + +Please look at `git log` and match the commit log style, which is very +similar to the +[Linux kernel](https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git). + +You may use `Signed-off-by`, but we're not requiring it. Running the test suite ---------------------- -Currently, OSTree uses - -To run just OSTree's tests: - - ./configure ... --enable-installed-tests - gnome-desktop-testing-runner -p 0 ostree/ - -Also, there is a regular: - - make check - -That runs a different set of tests. +OSTree uses both `make check` and supports the +[Installed Tests](https://wiki.gnome.org/GnomeGoals/InstalledTests) +model as well (if `--enable-installed-tests` is provided). Coding style ------------ From c276025466af4319d603fe8b534dd3463818a168 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 28 Mar 2016 13:08:32 +0100 Subject: [PATCH 16/69] test-xattrs: use TAP syntax to skip test Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/test-xattrs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-xattrs.sh b/tests/test-xattrs.sh index 6a83a0bc..2f2d99cc 100755 --- a/tests/test-xattrs.sh +++ b/tests/test-xattrs.sh @@ -21,7 +21,8 @@ set -euo pipefail touch test-xattrs if ! setfattr -n user.testvalue -v somevalue test-xattrs; then - exit 77 + echo "1..0 # SKIP: cannot run setfattr" + exit 0 fi echo "1..2" From 3e3755c497bd85e22b01829c5715119d46394687 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 11:06:05 +0100 Subject: [PATCH 17/69] various tests: skip if temp directory lacks xattr support Some autobuilder environments place the entire build chroot on tmpfs, so even /var/tmp might not have this. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/libtest.sh | 8 ++++++++ tests/test-basic-user.sh | 6 ++++-- tests/test-delta.sh | 2 ++ tests/test-demo-buildsystem.sh | 2 ++ tests/test-local-pull.sh | 6 ++++-- tests/test-prune.sh | 2 ++ tests/test-rofiles-fuse.sh | 3 +++ 7 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 06982d21..21de2e87 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -324,3 +324,11 @@ os_repository_new_commit () ${CMD_PREFIX} ostree --repo=${test_tmpdir}/testos-repo commit --add-metadata-string "version=${version}" -b testos/buildmaster/x86_64-runtime -s "Build" cd ${test_tmpdir} } + +skip_without_user_xattrs () { + touch test-xattrs + if ! setfattr -n user.testvalue -v somevalue test-xattrs; then + echo "1..0 # SKIP this test requires xattr support" + exit 0 + fi +} diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index f53de89a..42e6a864 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -19,10 +19,12 @@ set -euo pipefail -echo "1..1" - . $(dirname $0)/libtest.sh +skip_without_user_xattrs + +echo "1..1" + setup_test_repository "bare-user" echo "ok setup" diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 12f54c7d..a6fd6f88 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -21,6 +21,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh +skip_without_user_xattrs + bindatafiles="bash true ostree" morebindatafiles="false ls" diff --git a/tests/test-demo-buildsystem.sh b/tests/test-demo-buildsystem.sh index 500eac68..bffa59c4 100755 --- a/tests/test-demo-buildsystem.sh +++ b/tests/test-demo-buildsystem.sh @@ -26,6 +26,8 @@ fi . $(dirname $0)/libtest.sh +skip_without_user_xattrs + echo "1..1" # Run "triggers" like ldconfig, gtk-update-icon-cache, etc. diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index a9beb083..50ecbcb7 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -19,10 +19,12 @@ set -euo pipefail -echo "1..1" - . $(dirname $0)/libtest.sh +skip_without_user_xattrs + +echo "1..1" + setup_test_repository "archive-z2" echo "ok setup" diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 7184ea9c..e1796a33 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -21,6 +21,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh +skip_without_user_xattrs + setup_fake_remote_repo1 "archive-z2" echo '1..2' diff --git a/tests/test-rofiles-fuse.sh b/tests/test-rofiles-fuse.sh index 444fbce2..ba45959e 100755 --- a/tests/test-rofiles-fuse.sh +++ b/tests/test-rofiles-fuse.sh @@ -25,6 +25,9 @@ if ! fusermount --version >/dev/null 2>&1; then fi . $(dirname $0)/libtest.sh + +skip_without_user_xattrs + setup_test_repository "bare-user" echo "1..5" From f8bef792cb9cdd8b9cd174dad5856727db3b9bad Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Mon, 28 Mar 2016 13:46:23 +0100 Subject: [PATCH 18/69] Symlink libreaddir-rand.so into tests directory This means it can be LD_PRELOADed during build-time testing. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- .gitignore | 1 + Makefile-decls.am | 3 +++ Makefile-tests.am | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index d241cab6..f37ca48f 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ _build /test-remote-add.test /test-setuid.test /test-xattrs.test +/tests/libreaddir-rand.so test-varint test*.test *.trs diff --git a/Makefile-decls.am b/Makefile-decls.am index a5c9d503..eefffd64 100644 --- a/Makefile-decls.am +++ b/Makefile-decls.am @@ -49,3 +49,6 @@ GITIGNOREFILES = # This is a special facility to chain together hooks easily INSTALL_DATA_HOOKS = install-data-hook: $(INSTALL_DATA_HOOKS) + +ALL_LOCAL_RULES = +all-local: $(ALL_LOCAL_RULES) diff --git a/Makefile-tests.am b/Makefile-tests.am index ce562265..c92b0be1 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -216,6 +216,10 @@ EXTRA_DIST += \ tests/gpg-verify-data/trustdb.gpg \ tests/gpg-verify-data/gpg.conf +tests-libreaddir-rand-so-symlink: + ln -fns ../.libs/libreaddir-rand.so tests +ALL_LOCAL_RULES += tests-libreaddir-rand-so-symlink + # Unfortunately the glib test data APIs don't actually handle # non-recursive Automake, so we change our code to canonically look # for tests/ which is just a symlink when installed. From 47fd5c74f1428d4cc02ff0061a4958c4b714e852 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 08:24:56 +0100 Subject: [PATCH 19/69] tap-test: clean up temporary test directories as intended The script created ./.testtmp but looked for ./.test, which isn't going to work. This means the various "ostree trivial-httpd --autoexit" processes actually exit, because their web roots are cleaned up now. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- buildutil/tap-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutil/tap-test b/buildutil/tap-test index e7914541..6b2eb5c1 100755 --- a/buildutil/tap-test +++ b/buildutil/tap-test @@ -13,7 +13,7 @@ touch ${tempdir}/.testtmp function cleanup () { if test -n "${TEST_SKIP_CLEANUP:-}"; then echo "Skipping cleanup of ${tempdir}" - else if test -f ${tempdir}/.test; then + else if test -f ${tempdir}/.testtmp; then rm "${tempdir}" -rf fi fi From b25ddd29ab8eb56bc1022dd38758e54dc4b2ccbe Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 08:27:32 +0100 Subject: [PATCH 20/69] In tests that use gpg, terminate the gpg-agent after testing Otherwise we leak those processes. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/test-commit-sign.sh | 1 + tests/test-gpg-signed-commit.sh | 2 ++ tests/test-pull-mirror-summary.sh | 2 ++ tests/test-pull-summary-sigs.sh | 2 ++ tests/test-remote-gpg-import.sh | 1 + 5 files changed, 8 insertions(+) diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh index 8d52bcaf..1b153e9a 100755 --- a/tests/test-commit-sign.sh +++ b/tests/test-commit-sign.sh @@ -132,5 +132,6 @@ if ${CMD_PREFIX} ostree --repo=repo show main | grep -o 'Found [[:digit:]] signa fi rm -rf repo gnomerepo-files +gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye echo "ok" diff --git a/tests/test-gpg-signed-commit.sh b/tests/test-gpg-signed-commit.sh index a0bf9832..096c0e1c 100755 --- a/tests/test-gpg-signed-commit.sh +++ b/tests/test-gpg-signed-commit.sh @@ -78,4 +78,6 @@ if ${OSTREE} show test2 | grep -o 'Found [[:digit:]] signature'; then assert_not_reached fi +gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye + echo "ok" diff --git a/tests/test-pull-mirror-summary.sh b/tests/test-pull-mirror-summary.sh index 20707a13..d5e27fc8 100755 --- a/tests/test-pull-mirror-summary.sh +++ b/tests/test-pull-mirror-summary.sh @@ -121,3 +121,5 @@ echo "ok pull mirror with invalid summary sig and no verification" # assert_file_has_content deltas.txt "${origmain}-${newmain}" # echo "ok pull mirror with signed summary covering static deltas" + +gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index 202efdae..dd2fcb0f 100755 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -133,3 +133,5 @@ assert_file_has_content summary.txt "Good signature from \"Ostree Tester static-deltas.txt assert_file_has_content static-deltas.txt \ $(${OSTREE} --repo=repo rev-parse origin:main) + +gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index bb0c4029..aa90eb89 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -143,4 +143,5 @@ if ${OSTREE} pull R2:main >/dev/null 2>&1; then fi ${OSTREE} pull R3:main >/dev/null +gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye echo "ok" From 2b9032f016a1ee0bfa84df32d02057086bbe33fd Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 08:42:09 +0100 Subject: [PATCH 21/69] .gitignore: update Closes: #232 Approved by: cgwalters --- .gitignore | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index f37ca48f..0eb6981b 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,12 @@ ABOUT-NLS _build /build-aux/test-driver +/buildutil/gtk-doc.m4 +/buildutil/libtool.m4 +/buildutil/ltoptions.m4 +/buildutil/ltsugar.m4 +/buildutil/ltversion.m4 +/buildutil/lt~obsolete.m4 /doc/ostree-decl.txt /doc/ostree-overrides.txt @@ -51,12 +57,20 @@ _build /doc/xml /doc/ostree.1 +/man/*.1 +/man/*.5 + /ostree /ostree-prepare-root /ostree-remount /OSTree-1.0.gir +/rofiles-fuse + /src/libostree/ostree-1.pc +/src/libostree/ostree-enumtypes.c +/src/libostree/ostree-enumtypes.h +/src/ostree/parse-datetime.c /test-admin-deploy-1.test /test-admin-deploy-2.test @@ -71,6 +85,10 @@ _build /test-setuid.test /test-xattrs.test /tests/libreaddir-rand.so +/tests/test-basic-c +/tests/test-libarchive-import +/tests/test-lzma +/tests/test-sysroot-c test-varint test*.test *.trs From 07aa8e1c76463e3de0ffc79d0271773ababf05b4 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 09:02:23 +0100 Subject: [PATCH 22/69] Load g-i bindings from builddir during build-time testing Previously, the build-time tests would only pass if the g-i bindings to OSTree were already installed, with a reasonably similar version. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- Makefile-tests.am | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index c92b0be1..4241ddb6 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -25,7 +25,10 @@ include $(top_srcdir)/buildutil/glib-tap.mk TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ SRCDIR=$$(cd $(top_srcdir) && pwd) \ BUILDDIR=$$(cd $(top_builddir) && pwd) \ - PATH=$$(cd $(top_builddir) && pwd):$${PATH} + GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd) \ + LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd) \ + PATH=$$(cd $(top_builddir) && pwd):$${PATH} \ + $(NULL) uninstalled_test_scripts = tests/test-abi.sh From 1dd85513e540ef0b42f2fee7238fa1f710eaa4ca Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 09:48:18 +0100 Subject: [PATCH 23/69] tests/admin-test.sh: this is a bash script, not a POSIX sh script The "function foo()" syntax is bash-specific, and Colin indicated in PR #226 that he prefers to require bash rather than trying to support every POSIX shell. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/admin-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 3a04a69c..43dcbb55 100755 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright (C) 2011,2014 Colin Walters # # This library is free software; you can redistribute it and/or From d458399615325b39de03084eff1194d5852fe9f4 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 10:17:02 +0100 Subject: [PATCH 24/69] Force libreaddir-rand to be a shared library If installed-tests are disabled, it would normally be a static (convenience) library, which isn't something we can LD_PRELOAD. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- Makefile-tests.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile-tests.am b/Makefile-tests.am index 4241ddb6..6016d03c 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -131,6 +131,9 @@ libreaddir_rand_la_SOURCES = tests/readdir-rand.c libreaddir_rand_la_CFLAGS = $(OT_INTERNAL_GIO_UNIX_CFLAGS) libreaddir_rand_la_LIBADD = $(OT_INTERNAL_GIO_UNIX_LIBS) libreaddir_rand_la_LDFLAGS = -avoid-version +if !ENABLE_INSTALLED_TESTS +libreaddir_rand_la_LDFLAGS += -rpath $(abs_builddir) +endif test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ From e1ce859368b41c863fe096df84f1ff33e90d725f Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 13:44:27 +0100 Subject: [PATCH 25/69] Skip tests that run rofiles-fuse if /dev/fuse or /etc/mtab unavailable Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/libtest.sh | 17 +++++++++++++++++ tests/test-demo-buildsystem.sh | 6 +----- tests/test-rofiles-fuse.sh | 6 +----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 21de2e87..3e4c4bd6 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -332,3 +332,20 @@ skip_without_user_xattrs () { exit 0 fi } + +skip_without_fuse () { + if ! fusermount --version >/dev/null 2>&1; then + echo "1..0 # SKIP no fusermount" + exit 0 + fi + + if ! [ -w /dev/fuse ]; then + echo "1..0 # SKIP no write access to /dev/fuse" + exit 0 + fi + + if ! [ -e /etc/mtab ]; then + echo "1..0 # SKIP no /etc/mtab" + exit 0 + fi +} diff --git a/tests/test-demo-buildsystem.sh b/tests/test-demo-buildsystem.sh index bffa59c4..97915f94 100755 --- a/tests/test-demo-buildsystem.sh +++ b/tests/test-demo-buildsystem.sh @@ -19,13 +19,9 @@ set -euo pipefail -if ! fusermount --version >/dev/null 2>&1; then - echo "1..0 # SKIP no fusermount" - exit 0 -fi - . $(dirname $0)/libtest.sh +skip_without_fuse skip_without_user_xattrs echo "1..1" diff --git a/tests/test-rofiles-fuse.sh b/tests/test-rofiles-fuse.sh index ba45959e..d021df09 100755 --- a/tests/test-rofiles-fuse.sh +++ b/tests/test-rofiles-fuse.sh @@ -19,13 +19,9 @@ set -euo pipefail -if ! fusermount --version >/dev/null 2>&1; then - echo "1..0 # SKIP no fusermount" - exit 0 -fi - . $(dirname $0)/libtest.sh +skip_without_fuse skip_without_user_xattrs setup_test_repository "bare-user" From 7835fcdc682ef02654ae6aee98b45283478ec275 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 31 Mar 2016 14:07:01 +0100 Subject: [PATCH 26/69] test-pull-untrusted.sh: always corrupt a regular file, not a symlink test-pull-untrusted.sh would pass when run as root, but fail when run as testuser. It turned out that the way the files were stored in the repo when running as a testuser were different, which meant that a different .file object was chosen for corruption. Except that file turned out to be a symlink, so the echo "broke" actually just wrote to the no_such_file symlink target, thus keeping the actual symlink file's checksum the same and causing the pull-local to pass when it should have failed. [smcv: split this out of a larger commit, part of PR #231] Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/test-pull-untrusted.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test-pull-untrusted.sh b/tests/test-pull-untrusted.sh index 95f7ab93..53636224 100755 --- a/tests/test-pull-untrusted.sh +++ b/tests/test-pull-untrusted.sh @@ -39,6 +39,12 @@ echo "ok pull-local --untrusted didn't hardlink" # Corrupt repo for i in ${test_tmpdir}/repo/objects/*/*.file; do + + # make sure it's not a symlink + if [ -L $i ]; then + continue + fi + echo "corrupting $i" echo "broke" >> $i break; From 8cda8b6866f86ffdda8622c9108e949d0ed6c27f Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 31 Mar 2016 14:10:27 +0100 Subject: [PATCH 27/69] basic-test: commit with a non-empty subject [smcv: split out from a larger commit, part of PR #231; add commit message] Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/basic-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 1ae7e7a4..5519764a 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -114,7 +114,7 @@ assert_streq $($OSTREE log test2-custom-parent |grep '^commit' | wc -l) "3" echo "ok commit custom parent" cd ${test_tmpdir} -orphaned_rev=$($OSTREE commit --orphan -s '' $test_tmpdir/checkout-test2-4) +orphaned_rev=$($OSTREE commit --orphan -s "$(date)" $test_tmpdir/checkout-test2-4) $OSTREE ls ${orphaned_rev} >/dev/null $OSTREE prune --refs-only if $OSTREE ls ${orphaned_rev} 2>err.txt; then From 9dafc820241ed6e663ad4093660fc255f549aa26 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 14:32:55 +0100 Subject: [PATCH 28/69] Probe for GNU parallel more accurately moreutils parallel isn't compatible with the command-line syntax used here. Because it doesn't implement GNU-style --help and exits 1 when that option is given, this test was correctly skipped when using moreutils parallel, but only by mistake. moreutils parallel might conceivably gain --help and --version in future, but hopefully nothing incompatible with GNU parallel is going to gain a --gnu option. Also use the --gnu option to force the new command-line semantics; some versions optionally supported an incompatible command-line syntax taken from moreutils parallel. Signed-off-by: Simon McVittie Closes: #232 Approved by: cgwalters --- tests/test-admin-locking.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-admin-locking.sh b/tests/test-admin-locking.sh index 564295d2..01814f9f 100755 --- a/tests/test-admin-locking.sh +++ b/tests/test-admin-locking.sh @@ -25,8 +25,8 @@ set -euo pipefail setup_os_repository "archive-z2" "syslinux" # If parallel is not installed, skip the test -if ! parallel --help >/dev/null 2>&1; then - echo "1..0 # SKIP no /usr/bin/parallel" +if ! parallel --gnu /bin/true /dev/null 2>&1; then + echo "1..0 # SKIP GNU parallel not available" exit 0 fi @@ -42,7 +42,7 @@ echo "rev=${rev}" ${CMD_PREFIX} ostree admin deploy --karg=root=LABEL=MOO --karg=quiet --os=testos testos:testos/buildmaster/x86_64-runtime assert_has_dir sysroot/boot/ostree/testos-${bootcsum} -parallel_cmd=parallel +parallel_cmd="parallel --gnu" if parallel --help | grep -q -e --no-notice; then parallel_cmd="${parallel_cmd} --no-notice" fi From bdf24cdc049b421d6e5a982ee9f25502407dceca Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 31 Mar 2016 13:46:16 -0400 Subject: [PATCH 29/69] tests: Make failing to kill the GPG agent non-fatal It's not working for me in `make check` on a RHEL 7 Workstation, apparently because no GPG agent is spawned. I'm guessing this has something to do with the GPG version? The downside of this is we will be less likely to notice if GPG changes again and we start leaking agents like we're in The Matrix Reloaded. But the real solution to that is containers anyways. Closes: #233 Approved by: smcv --- tests/libtest.sh | 4 ++++ tests/test-commit-sign.sh | 2 +- tests/test-gpg-signed-commit.sh | 2 +- tests/test-pull-mirror-summary.sh | 2 +- tests/test-pull-summary-sigs.sh | 2 +- tests/test-remote-gpg-import.sh | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 3e4c4bd6..03b498aa 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -349,3 +349,7 @@ skip_without_fuse () { exit 0 fi } + +libtest_cleanup_gpg () { + gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye || true +} diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh index 1b153e9a..e469c0f9 100755 --- a/tests/test-commit-sign.sh +++ b/tests/test-commit-sign.sh @@ -132,6 +132,6 @@ if ${CMD_PREFIX} ostree --repo=repo show main | grep -o 'Found [[:digit:]] signa fi rm -rf repo gnomerepo-files -gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye +libtest_cleanup_gpg echo "ok" diff --git a/tests/test-gpg-signed-commit.sh b/tests/test-gpg-signed-commit.sh index 096c0e1c..fa42d677 100755 --- a/tests/test-gpg-signed-commit.sh +++ b/tests/test-gpg-signed-commit.sh @@ -78,6 +78,6 @@ if ${OSTREE} show test2 | grep -o 'Found [[:digit:]] signature'; then assert_not_reached fi -gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye +libtest_cleanup_gpg echo "ok" diff --git a/tests/test-pull-mirror-summary.sh b/tests/test-pull-mirror-summary.sh index d5e27fc8..49aa91ad 100755 --- a/tests/test-pull-mirror-summary.sh +++ b/tests/test-pull-mirror-summary.sh @@ -122,4 +122,4 @@ echo "ok pull mirror with invalid summary sig and no verification" # echo "ok pull mirror with signed summary covering static deltas" -gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye +libtest_cleanup_gpg diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index dd2fcb0f..40de0a66 100755 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -134,4 +134,4 @@ grep static-deltas summary.txt > static-deltas.txt assert_file_has_content static-deltas.txt \ $(${OSTREE} --repo=repo rev-parse origin:main) -gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye +libtest_cleanup_gpg diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index aa90eb89..8d155f79 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -143,5 +143,5 @@ if ${OSTREE} pull R2:main >/dev/null 2>&1; then fi ${OSTREE} pull R3:main >/dev/null -gpg-connect-agent --homedir ${test_tmpdir}/gpghome killagent /bye +libtest_cleanup_gpg echo "ok" From efb86ba9d32a038dd90f47f6e82f578e91b5fef5 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 18:29:44 +0100 Subject: [PATCH 30/69] libtest.sh: use G_TEST_SRCDIR, G_TEST_BUILDDIR to find resources This fixes the bug that in installed-tests that run testlib.sh under "bash -c" (i.e. the C and JS tests), $(dirname $0) is "." and we can't do the LD_PRELOAD correctly: ERROR: ld.so: object './libreaddir-rand.so' from LD_PRELOAD cannot be preloaded (cannot open shared object file): ignored. Similarly, those tests can't copy gpghome correctly. This also removes the confusing situation that SRCDIR in libtest.sh (which is ${top_srcdir}/tests) does not mean the same thing as SRCDIR in test-abi.sh (which is just ${top_srcdir}). Signed-off-by: Simon McVittie Closes: #234 Approved by: cgwalters --- tests/libtest.sh | 18 ++++++++++++++---- tests/test-archivez.sh | 2 +- tests/test-delta.sh | 4 ++-- tests/test-pull-archive-z.sh | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/libtest.sh b/tests/libtest.sh index 03b498aa..aaa553a0 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -17,7 +17,17 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -SRCDIR=$(dirname $0) +if [ -n "${G_TEST_SRCDIR:-}" ]; then + test_srcdir="${G_TEST_SRCDIR}/tests" +else + test_srcdir=$(dirname $0) +fi + +if [ -n "${G_TEST_BUILDDIR:-}" ]; then + test_builddir="${G_TEST_BUILDDIR}/tests" +else + test_builddir=$(dirname $0) +fi assert_not_reached () { echo $@ 1>&2; exit 1 @@ -54,7 +64,7 @@ export TEST_GPG_KEYID_3="DF444D67" # homedir in order to create lockfiles. Work around # this by copying locally. echo "Copying gpghome to ${test_tmpdir}" -cp -a ${SRCDIR}/gpghome ${test_tmpdir} +cp -a "${test_srcdir}/gpghome" ${test_tmpdir} export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted @@ -63,9 +73,9 @@ if test -n "${OT_TESTS_DEBUG:-}"; then fi if test -n "${OT_TESTS_VALGRIND:-}"; then - CMD_PREFIX="env G_SLICE=always-malloc valgrind -q --leak-check=full --num-callers=30 --suppressions=${SRCDIR}/ostree-valgrind.supp" + CMD_PREFIX="env G_SLICE=always-malloc valgrind -q --leak-check=full --num-callers=30 --suppressions=${test_srcdir}/ostree-valgrind.supp" else - CMD_PREFIX="env LD_PRELOAD=${SRCDIR}/libreaddir-rand.so" + CMD_PREFIX="env LD_PRELOAD=${test_builddir}/libreaddir-rand.so" fi assert_streq () { diff --git a/tests/test-archivez.sh b/tests/test-archivez.sh index 999157de..b8793284 100755 --- a/tests/test-archivez.sh +++ b/tests/test-archivez.sh @@ -25,7 +25,7 @@ echo '1..11' setup_test_repository "archive-z2" -. ${SRCDIR}/archive-test.sh +. ${test_srcdir}/archive-test.sh cd ${test_tmpdir} mkdir repo2 diff --git a/tests/test-delta.sh b/tests/test-delta.sh index a6fd6f88..411e9b04 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -130,9 +130,9 @@ assert_streq "${totalsize_orig}" "${totalsize_swapped}" echo 'ok generate + show endian swapped' -tar xf ${SRCDIR}/pre-endian-deltas-repo-big.tar.xz +tar xf ${test_srcdir}/pre-endian-deltas-repo-big.tar.xz mv pre-endian-deltas-repo{,-big} -tar xf ${SRCDIR}/pre-endian-deltas-repo-little.tar.xz +tar xf ${test_srcdir}/pre-endian-deltas-repo-little.tar.xz mv pre-endian-deltas-repo{,-little} legacy_origrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main^) legacy_newrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main) diff --git a/tests/test-pull-archive-z.sh b/tests/test-pull-archive-z.sh index 2ea23871..66f8873e 100755 --- a/tests/test-pull-archive-z.sh +++ b/tests/test-pull-archive-z.sh @@ -23,4 +23,4 @@ set -euo pipefail setup_fake_remote_repo1 "archive-z2" -. ${SRCDIR}/pull-test.sh +. ${test_srcdir}/pull-test.sh From 839628b3fa9e7d6bec0955b973e519a8988e264c Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 18:30:52 +0100 Subject: [PATCH 31/69] test-abi: use G_TEST_SRCDIR, G_TEST_BUILDDIR There's no need to invent new variables for these. Signed-off-by: Simon McVittie Closes: #234 Approved by: cgwalters --- Makefile-tests.am | 2 -- tests/test-abi.sh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 6016d03c..1bc1bc90 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -23,8 +23,6 @@ include $(top_srcdir)/buildutil/glib-tap.mk # include the builddir in $PATH so we find our just-built ostree # binary. TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \ - SRCDIR=$$(cd $(top_srcdir) && pwd) \ - BUILDDIR=$$(cd $(top_builddir) && pwd) \ GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd) \ LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd) \ PATH=$$(cd $(top_builddir) && pwd):$${PATH} \ diff --git a/tests/test-abi.sh b/tests/test-abi.sh index a49f8d4b..62af3652 100755 --- a/tests/test-abi.sh +++ b/tests/test-abi.sh @@ -21,8 +21,8 @@ set -euo pipefail echo '1..1' -grep ' ostree_[A-Za-z0-9_]*;' ${SRCDIR}/src/libostree/libostree.sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt -eu-readelf -a ${BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9_.]*,\1,' |sort -u > found-symbols.txt +grep ' ostree_[A-Za-z0-9_]*;' ${G_TEST_SRCDIR}/src/libostree/libostree.sym | sed -e 's,^ *\([A-Za-z0-9_]*\);,\1,' | sort -u > expected-symbols.txt +eu-readelf -a ${G_TEST_BUILDDIR}/.libs/libostree-1.so | grep 'FUNC.*GLOBAL.*DEFAULT.*@@LIBOSTREE_' | sed -e 's,^.* \(ostree_[A-Za-z0-9_]*\)@@LIBOSTREE_[0-9_.]*,\1,' |sort -u > found-symbols.txt diff -u expected-symbols.txt found-symbols.txt echo 'ok' From 9c4babf316cfbfad61cfcd0ab8fd5e77e9df13be Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 17:47:36 +0100 Subject: [PATCH 32/69] test-xattrs: sync how this is skipped with test-rofiles-fuse If we touch test-xattrs before sourcing libtest.sh, we get: test tmpdir=/tmp/test-tmp-ostree_test-xattrs.sh.test-HSEXEY is not empty; run this test via `make check TESTS=`, not directly Signed-off-by: Simon McVittie Closes: #234 Approved by: cgwalters --- tests/test-xattrs.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test-xattrs.sh b/tests/test-xattrs.sh index 2f2d99cc..cdc06e87 100755 --- a/tests/test-xattrs.sh +++ b/tests/test-xattrs.sh @@ -19,16 +19,12 @@ set -euo pipefail -touch test-xattrs -if ! setfattr -n user.testvalue -v somevalue test-xattrs; then - echo "1..0 # SKIP: cannot run setfattr" - exit 0 -fi +. $(dirname $0)/libtest.sh + +skip_without_user_xattrs echo "1..2" -. $(dirname $0)/libtest.sh - setup_test_repository "archive-z2" cd ${test_tmpdir} From 18e9169d7ae76149833475c14f4c7147f6f91c9e Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 31 Mar 2016 17:52:57 +0100 Subject: [PATCH 33/69] libtest.sh: only check whether $(pwd) is empty once test-sysroot.js runs libtestExec() twice, one of which is after creating non-hidden directories in $(pwd), so this check needs to be skipped the second time. Signed-off-by: Simon McVittie Closes: #234 Approved by: cgwalters --- tests/libtest.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/libtest.sh b/tests/libtest.sh index aaa553a0..765e987d 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -41,8 +41,12 @@ test_tmpdir=$(pwd) if ! test -f .testtmp; then files=$(ls) if test -n "${files}"; then + ls -l assert_not_reached "test tmpdir=${test_tmpdir} is not empty; run this test via \`make check TESTS=\`, not directly" fi + # Remember that this is an acceptable test $(pwd), for the benefit of + # C and JS tests which may source this file again + touch .testtmp fi export G_DEBUG=fatal-warnings From 826c2149b8e1c7be7277d9e558680139e9fd0ab7 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 4 Apr 2016 15:25:39 +0200 Subject: [PATCH 34/69] manual: Fix a bunch of typos and docbookisms Closes: #238 Approved by: cgwalters --- docs/manual/adapting-existing.md | 22 +++++++++++----------- docs/manual/atomic-upgrades.md | 14 +++++++------- docs/manual/buildsystem-and-repos.md | 4 ++-- docs/manual/deployment.md | 8 ++++---- docs/manual/related-projects.md | 2 +- docs/manual/repo.md | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md index 8509d48e..77746e94 100644 --- a/docs/manual/adapting-existing.md +++ b/docs/manual/adapting-existing.md @@ -23,7 +23,7 @@ deployment. Because OSTree only preserves `/var` across upgrades (each deployment's chroot directory will be garbage collected eventually), you will need to choose how to handle other -toplevel writable directories specified by the [Filesystem Hierarchy Standard](http://www.pathname.com/fhs/") +toplevel writable directories specified by the [Filesystem Hierarchy Standard](http://www.pathname.com/fhs/). Your operating system may of course choose not to support some of these such as `/usr/local`, but following is the recommended set: @@ -37,9 +37,9 @@ recommended set: - `/tmp` → `/sysroot/tmp` Furthermore, since `/var` is empty by default, your operating system -will need to dynamically create the targets of -these at boot. A good way to do this is using `systemd-tmpfiles`, if -your OS uses systemd. For example: +will need to dynamically create the *targets* of these at boot. A +good way to do this is using `systemd-tmpfiles`, if your OS uses +systemd. For example: ``` d /var/log/journal 0755 root root - @@ -64,10 +64,10 @@ d /run/media 0755 root root - Particularly note here the double indirection of `/home`. By default, each deployment will share the global toplevel `/home` directory on the physical root filesystem. It is then up to higher levels of -management tools to keep /etc/passwd or -equivalent synchronized between operating systems. Each deployment -can easily be reconfigured to have its own home directory set simply -by making `/var/home` a real directory. +management tools to keep `/etc/passwd` or equivalent synchronized +between operating systems. Each deployment can easily be reconfigured +to have its own home directory set simply by making `/var/home` a real +directory. ## Booting and initramfs technology @@ -144,11 +144,11 @@ these new packages on top. A command like this: ``` ostree commit -b osname/releasename/description ---tree=ref=$osname/releasename/description +--tree=ref=$osname/$releasename/$description --tree=dir=/var/tmp/newpackages.13A8D0/ ``` -will create a new commit in the `$osname/releasename/description` +will create a new commit in the `$osname/$releasename/$description` branch. The OSTree SHA256 checksum of all the files in `/var/tmp/newpackages.13A8D0/` will be computed, but we will not re-checksum the present existing tree. In this layering model, @@ -156,4 +156,4 @@ earlier directories will take precedence, but files in later layers will silently override earlier layers. Then to actually deploy this tree for the next boot: -`ostree admin deploy osname/releasename/description` +`ostree admin deploy $osname/$releasename/$description` diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md index fa576734..b5f398d6 100644 --- a/docs/manual/atomic-upgrades.md +++ b/docs/manual/atomic-upgrades.md @@ -18,7 +18,7 @@ implements this. To begin a simple upgrade, OSTree fetches the contents of the ref from the remote server. Suppose we're tracking a ref named `exampleos/buildmaster/x86_64-runtime`. OSTree fetches the URL -`http://$example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`, +`http://example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`, which contains a SHA256 checksum. This determines the tree to deploy, and `/etc` will be merged from currently booted tree. @@ -35,7 +35,7 @@ we need to perform a deployment. As mentioned in the introduction, OSTree is also designed to allow a model where filesystem trees are computed on the client. It is -completely agnostic as to how those trees are generated; hey could be +completely agnostic as to how those trees are generated; they could be computed with traditional packages, packages with post-deployment scripts on top, or built by developers directly from revision control locally, etc. @@ -58,7 +58,7 @@ Given a commit to deploy, OSTree first allocates a directory for it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. The `$serial` is normally 0, but if a given commit is deployed more than once, it will be incremented. -his is supported because the previous deployment may have +This is supported because the previous deployment may have configuration in `/etc` that we do not want to use or overwrite. Now that we have a deployment directory, a 3-way merge is @@ -94,7 +94,7 @@ collected at any point. ## The /ostree/boot directory -However, we want to optimize for the case where we the set of +However, we want to optimize for the case where the set of kernel/initramfs pairs is the same between both the old and new deployment lists. This happens when doing an upgrade that does not include the kernel; think of a simple translation update. OSTree @@ -106,11 +106,11 @@ automatically remount read-write just for the portion of time necessary to update the bootloader configuration. To implement this, OSTree also maintains the directory -`/ostree/boot.bootversion`, which is a set +`/ostree/boot.$bootversion`, which is a set of symbolic links to the deployment directories. The -bootversion here must match the version of +`$bootversion` here must match the version of `/boot`. However, in order to allow atomic transitions of -this directory, this is also a swapped directory, +*this* directory, this is also a swapped directory, so just like `/boot`, it has a version of `0` or `1` appended. Each bootloader entry has a special `ostree=` argument which refers to diff --git a/docs/manual/buildsystem-and-repos.md b/docs/manual/buildsystem-and-repos.md index 2da3d8d7..1f14b181 100644 --- a/docs/manual/buildsystem-and-repos.md +++ b/docs/manual/buildsystem-and-repos.md @@ -63,7 +63,7 @@ But let's discuss building our own. If you're just experimenting, it's quite easy to start with the command line. We'll assume for this purpose that you have a build process that outputs a directory tree - we'll call this tool `$pkginstallroot` (which could be `yum ---installroot` or `dbootstrap`, etc.). +--installroot` or `debootstrap`, etc.). Your initial prototype is going to look like: @@ -132,7 +132,7 @@ the desired version). Now, to construct our final tree: ``` -rm exampleos-build -rf +rm -rf exampleos-build for package in bash systemd; do ostree --repo=build-repo checkout -U --union exampleos/x86_64/${package} exampleos-build done diff --git a/docs/manual/deployment.md b/docs/manual/deployment.md index f1172959..dc77809c 100644 --- a/docs/manual/deployment.md +++ b/docs/manual/deployment.md @@ -54,9 +54,9 @@ to avoid computing checksums on the client by default. The deployment should not have a traditional UNIX `/etc`; instead, it should include `/usr/etc`. This is the "default configuration". When OSTree creates a deployment, it performs a 3-way merge using the -old default configuration, the active system's -`/etc`, and the new default configuration. In the final filesystem -tree for a deployment then, `/etc` is a regular writable directory. +*old* default configuration, the active system's `/etc`, and the new +default configuration. In the final filesystem tree for a deployment +then, `/etc` is a regular writable directory. Besides the exceptions of `/var` and `/etc` then, the rest of the contents of the tree are checked out as hard links into the @@ -87,4 +87,4 @@ deployment. At present, not all bootloaders implement the BootLoaderSpec, so OSTree contains code for some of these to regenerate native config -files (such as `/boot/syslinux/syslinux.conf` based on the entries. +files (such as `/boot/syslinux/syslinux.conf`) based on the entries. diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index 48bc3f1c..b53ba792 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -18,7 +18,7 @@ date, and relatively agnostic. Broadly speaking, projects in this area fall into two camps; either a tool to snapshot systems on the client side (dpkg/rpm + BTRFS/LVM), or a tool to compose on a server and replicate (ChromiumOS, Clear -Linux). OSTree is flexible enough to do both. +Linux). OSTree is flexible enough to do both. ## Combining dpkg/rpm + (BTRFS/LVM) diff --git a/docs/manual/repo.md b/docs/manual/repo.md index e8d94b4b..d3be549c 100644 --- a/docs/manual/repo.md +++ b/docs/manual/repo.md @@ -87,7 +87,7 @@ two different generated filesystem trees. In this example, the "runtime" tree contains just enough to run a basic system, and "devel-debug" contains all of the developer tools and debuginfo. -The `ostree` supports a simple syntax using the carat `^` to refer to +The `ostree` supports a simple syntax using the caret `^` to refer to the parent of a given commit. For example, `exampleos/buildmaster/x86_64-runtime^` refers to the previous build, and `exampleos/buildmaster/x86_64-runtime^^` refers to the one before From 4e815484470fb22a1b0dc52af0193d5ce1a3caf7 Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Fri, 1 Apr 2016 13:51:18 +0200 Subject: [PATCH 35/69] Introducing ostree-grub-generator ostree-grub-generator can be used to customize the generated grub.cfg file. Compile time decision ostree-grub-generator vs grub2-mkconfig can be overwritten with the OSTREE_GRUB2_EXEC envvar - useful for auto tests and OS installers. Why this alternative approach: 1) The current approach is less flexible than using a custom 'ostree-grub-generator' script. Each system can adjust this script for its needs, instead of using the hardcoded values from ostree-bootloader-grub2.c. 2) Too much overhead on embedded to generate grub.cfg via /etc/grub.d/ configuration files. It is still possible to do so, even with this patch applied. No need to install grub2 package on a target device. 3) The grub2-mkconfig code path has other issues: https://bugzilla.gnome.org/show_bug.cgi?id=761180 Task: https://bugzilla.gnome.org/show_bug.cgi?id=762220 Closes: #228 Approved by: cgwalters --- Makefile-boot.am | 15 ++- Makefile-tests.am | 4 +- configure.ac | 22 ++-- src/boot/{ => grub2}/grub2-15_ostree | 0 src/boot/grub2/ostree-grub-generator | 101 ++++++++++++++++++ src/libostree/ostree-bootloader-grub2.c | 24 ++++- tests/admin-test.sh | 15 ++- ...ck.py => bootloader-entries-crosscheck.py} | 5 + tests/libtest.sh | 17 +++ tests/ostree-grub-generator | 1 + tests/test-admin-deploy-grub2.sh | 6 +- 11 files changed, 187 insertions(+), 23 deletions(-) rename src/boot/{ => grub2}/grub2-15_ostree (100%) create mode 100644 src/boot/grub2/ostree-grub-generator rename tests/{syslinux-entries-crosscheck.py => bootloader-entries-crosscheck.py} (96%) create mode 120000 tests/ostree-grub-generator diff --git a/Makefile-boot.am b/Makefile-boot.am index 01ed7470..a2cdc28f 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -1,4 +1,4 @@ -# Makefile for dracut module +# Makefile for boot module # # Copyright (C) 2013 Colin Walters # @@ -39,14 +39,17 @@ systemdsystemunit_DATA = src/boot/ostree-prepare-root.service \ src/boot/ostree-remount.service endif -dist_pkglibexec_SCRIPTS = src/boot/grub2-15_ostree - -if BUILDOPT_GRUB2 +if !BUILDOPT_BUILTIN_GRUB2_MKCONFIG +# We're using the system grub2-mkconfig generator +libexec_SCRIPTS = src/boot/grub2/grub2-15_ostree install-grub2-config-hook: mkdir -p $(DESTDIR)$(grub2configdir) - ln -sf $(pkglibexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree + ln -sf $(libexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree grub2configdir = $(sysconfdir)/grub.d INSTALL_DATA_HOOKS += install-grub2-config-hook +else +# We're using our internal generator +libexec_SCRIPTS = src/boot/grub2/ostree-grub-generator endif EXTRA_DIST += src/boot/dracut/module-setup.sh \ @@ -54,4 +57,6 @@ EXTRA_DIST += src/boot/dracut/module-setup.sh \ src/boot/mkinitcpio/ostree \ src/boot/ostree-prepare-root.service \ src/boot/ostree-remount.service \ + src/boot/grub2/grub2-15_ostree \ + src/boot/grub2/ostree-grub-generator \ $(NULL) diff --git a/Makefile-tests.am b/Makefile-tests.am index 1bc1bc90..b7ff268d 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -59,6 +59,7 @@ test_scripts = \ tests/test-admin-deploy-switch.sh \ tests/test-admin-deploy-etcmerge-cornercases.sh \ tests/test-admin-deploy-uboot.sh \ + tests/test-admin-deploy-grub2.sh \ tests/test-admin-instutil-set-kargs.sh \ tests/test-admin-upgrade-not-backwards.sh \ tests/test-admin-pull-deploy-commit.sh \ @@ -97,7 +98,8 @@ installed_test_data = tests/archive-test.sh \ tests/pre-endian-deltas-repo-little.tar.xz \ $(NULL) -test_extra_scripts = tests/syslinux-entries-crosscheck.py +test_extra_scripts = tests/bootloader-entries-crosscheck.py \ + tests/ostree-grub-generator # We can't use nobase_ as we need to strip off the tests/, can't # use plain installed_ as we do need the gpghome/ prefix. diff --git a/configure.ac b/configure.ac index 86df2a41..7dfcd500 100644 --- a/configure.ac +++ b/configure.ac @@ -251,11 +251,20 @@ AS_IF([test "x$with_dracut" = "xyes" || test "x$with_mkinitcpio" = "xyes"], [ ]) AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_systemd = xyes) -AC_ARG_WITH(grub2, - AS_HELP_STRING([--with-grub2], - [Install grub2 hook (default: yes)]),, - [with_grub2=yes]) -AM_CONDITIONAL(BUILDOPT_GRUB2, test x$with_grub2 = xyes) +AC_ARG_WITH(builtin-grub2-mkconfig, + AS_HELP_STRING([--with-builtin-grub-mkconfig], + [Use a builtin minimal grub2-mkconfig to generate a GRUB2 configuration file (default: no)]),, + [with_builtin_grub2_mkconfig=no]) +AM_CONDITIONAL(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, test x$with_builtin_grub2_mkconfig = xyes) +AM_COND_IF(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, + AC_DEFINE([USE_BUILTIN_GRUB2_MKCONFIG], 1, [Define if using internal ostree-grub-generator]), +[ + # Otherwise, look for the path to the system generator. On some + # distributions GRUB2 *-mkconfig executable has 'grub2' prefix and + # on some 'grub'. + AC_CHECK_PROG(GRUB2_MKCONFIG, grub2-mkconfig, grub2-mkconfig, grub-mkconfig) + AC_DEFINE_UNQUOTED([GRUB2_MKCONFIG_PATH], ["$GRUB2_MKCONFIG"], [The system grub2-mkconfig executible name]) +]) dnl for tests AS_IF([test "x$found_introspection" = xyes], [ @@ -292,7 +301,8 @@ echo " api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs dracut: $with_dracut - mkinitcpio: $with_mkinitcpio" + mkinitcpio: $with_mkinitcpio + builtin grub2-mkconfig (instead of system): $with_builtin_grub2_mkconfig" AS_IF([test "x$with_systemd" = "xyes"], [ echo " systemd unit dir: $with_systemdsystemunitdir" ]) diff --git a/src/boot/grub2-15_ostree b/src/boot/grub2/grub2-15_ostree similarity index 100% rename from src/boot/grub2-15_ostree rename to src/boot/grub2/grub2-15_ostree diff --git a/src/boot/grub2/ostree-grub-generator b/src/boot/grub2/ostree-grub-generator new file mode 100644 index 00000000..5673b264 --- /dev/null +++ b/src/boot/grub2/ostree-grub-generator @@ -0,0 +1,101 @@ +#!/bin/sh + +# To use a custrom script for generating grub.cfg, set the --with-grub2-mkconfig=no +# configure switch when configuring and building OSTree. +# +# This script is called by ostree/src/libostree/ostree-bootloader-grub2.c whenever +# boot loader configuration file needs to be updated. It can be used as a template +# for a custom grub.cfg generator. What to consider when writing a custom grub.cfg +# generator: +# +# - The populate_menu() function converts boot loader entries as defined by +# https://www.freedesktop.org/wiki/Specifications/BootLoaderSpec/ into GRUB2 +# menuentry sections. This is the core logic that is required by OSTree +# based system. +# +# - Embedded systems: Be aware that this script is executed not only on a host machine by OS +# installer, but also on a target device, thus think about shell portability. A target device +# for example might be using busybox with a limited shell. +# +# Feel free to edit this script to fit your requirements. + +set -e + +script=$(basename ${0}) +# Atomically safe location where to generete grub.cfg when executing system upgrade. +new_grub2_cfg=${2} +entries_path=$(dirname $new_grub2_cfg)/entries + +read_config() +{ + config_file=${entries_path}/${1} + title="" + initrd="" + options="" + linux="" + + while read -r line + do + record=$(echo ${line} | cut -f 1 -d ' ') + value=$(echo ${line} | cut -s -f2- -d ' ') + case "${record}" in + "title") + title=${value} + ;; + "initrd") + initrd=${value} + ;; + "linux") + linux=${value} + ;; + "options") + options=${value} + ;; + esac + done < ${config_file} + + if [ -z "${title}" ]; then + title="(Untitled)" + fi +} + +populate_menu() +{ + boot_prefix="${OSTREE_BOOT_PARTITION}" + for config in $(ls ${entries_path}); do + read_config ${config} + menu="${menu}menuentry '${title}' {\n" + menu="${menu}\t linux ${boot_prefix}${linux} ${options}\n" + menu="${menu}\t initrd ${boot_prefix}${initrd}\n" + menu="${menu}}\n\n" + done + # The printf command seems to be more reliable across shells for special character (\n, \t) evaluation + printf "$menu" >> ${new_grub2_cfg} +} + +populate_warning() +{ +cat >> ${new_grub2_cfg} <> ${new_grub2_cfg} <sysroot) == NULL +#ifdef USE_BUILTIN_GRUB2_MKCONFIG + use_system_grub2_mkconfig = FALSE; +#endif + /* Autotests can set this envvar to select which code path to test, useful for OS installers as well */ + grub_exec = g_getenv ("OSTREE_GRUB2_EXEC"); + if (grub_exec) + { + if (g_str_has_suffix (grub_exec, GRUB2_MKCONFIG_PATH)) + use_system_grub2_mkconfig = TRUE; + else + use_system_grub2_mkconfig = FALSE; + } + else + grub_exec = use_system_grub2_mkconfig ? GRUB2_MKCONFIG_PATH : LIBEXECDIR "/ostree-grub-generator"; + + if (use_system_grub2_mkconfig && ostree_sysroot_get_booted_deployment (self->sysroot) == NULL && g_file_has_parent (self->sysroot->path, NULL)) { g_autoptr(GPtrArray) deployments = NULL; @@ -318,6 +335,9 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader, * * In the case of an installer, use the first deployment root (which * will most likely be the only one. + * + * This all only applies if we're not using the builtin + * generator, which handles being run outside of the root. */ tool_deployment_root = ostree_sysroot_get_deployment_directory (self->sysroot, tool_deployment); grub2_mkconfig_chroot = g_file_get_path (tool_deployment_root); @@ -361,7 +381,7 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader, Upstream is fixed though. */ proc = g_subprocess_launcher_spawn (launcher, error, - "grub2-mkconfig", "-o", + grub_exec, "-o", gs_file_get_path_cached (new_config_path), NULL); diff --git a/tests/admin-test.sh b/tests/admin-test.sh index 43dcbb55..dcc34068 100755 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -21,10 +21,17 @@ set -euo pipefail echo "1..16" function validate_bootloader() { - (cd ${test_tmpdir}; - if test -f sysroot/boot/syslinux/syslinux.cfg; then - $(dirname $0)/syslinux-entries-crosscheck.py sysroot - fi) + cd ${test_tmpdir}; + bootloader="" + if test -f sysroot/boot/syslinux/syslinux.cfg; then + bootloader="syslinux" + elif test -f sysroot/boot/grub2/grub.cfg; then + bootloader="grub2" + fi + if test -n "${bootloader}"; then + $(dirname $0)/bootloader-entries-crosscheck.py sysroot ${bootloader} + fi + cd - } orig_mtime=$(stat -c '%.Y' sysroot/ostree/deploy) diff --git a/tests/syslinux-entries-crosscheck.py b/tests/bootloader-entries-crosscheck.py similarity index 96% rename from tests/syslinux-entries-crosscheck.py rename to tests/bootloader-entries-crosscheck.py index 8b58ed43..38e8e451 100755 --- a/tests/syslinux-entries-crosscheck.py +++ b/tests/bootloader-entries-crosscheck.py @@ -25,9 +25,14 @@ if len(sys.argv) == 1: else: sysroot = sys.argv[1] +bootloader = sys.argv[2] loaderpath = sysroot + '/boot/loader/entries' syslinuxpath = sysroot + '/boot/syslinux/syslinux.cfg' +if bootloader == "grub2": + sys.stdout.write('GRUB2 configuration validation not implemented.\n') + sys.exit(0) + def fatal(msg): sys.stderr.write(msg) sys.stderr.write('\n') diff --git a/tests/libtest.sh b/tests/libtest.sh index 765e987d..572f023c 100755 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -229,6 +229,20 @@ setup_os_boot_uboot() { ln -s loader/uEnv.txt sysroot/boot/uEnv.txt } +setup_os_boot_grub2() { + grub2_options=$1 + mkdir -p sysroot/boot/grub2/ + ln -s ../loader/grub.cfg sysroot/boot/grub2/grub.cfg + export OSTREE_BOOT_PARTITION="/boot" + case "$grub2_options" in + *ostree-grub-generator*) + cp ${test_srcdir}/ostree-grub-generator ${test_tmpdir} + chmod +x ${test_tmpdir}/ostree-grub-generator + export OSTREE_GRUB2_EXEC=${test_tmpdir}/ostree-grub-generator + ;; + esac +} + setup_os_repository () { mode=$1 bootmode=$2 @@ -301,6 +315,9 @@ EOF "uboot") setup_os_boot_uboot ;; + *grub2*) + setup_os_boot_grub2 "${bootmode}" + ;; esac cd ${test_tmpdir} diff --git a/tests/ostree-grub-generator b/tests/ostree-grub-generator new file mode 120000 index 00000000..587281cd --- /dev/null +++ b/tests/ostree-grub-generator @@ -0,0 +1 @@ +../src/boot/grub2/ostree-grub-generator \ No newline at end of file diff --git a/tests/test-admin-deploy-grub2.sh b/tests/test-admin-deploy-grub2.sh index 8da294d7..6109a950 100755 --- a/tests/test-admin-deploy-grub2.sh +++ b/tests/test-admin-deploy-grub2.sh @@ -21,11 +21,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..1" - # Exports OSTREE_SYSROOT so --sysroot not needed. -setup_os_repository "archive-z2" "grub2" - -echo "ok setup" +setup_os_repository "archive-z2" "grub2 ostree-grub-generator" . $(dirname $0)/admin-test.sh From eabb4e3545ee51f52d8ad9b09d2ffa918d49ab62 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 4 Apr 2016 14:26:53 +0200 Subject: [PATCH 36/69] pull: Don't try to cache summaries for pull-local Not only does this not make sense from a performance perspective, but it also doesn't work because we can't use a url as a path element. Closes: #237 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 26284806..238fdf65 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2107,12 +2107,14 @@ ostree_repo_pull_with_options (OstreeRepo *self, soup_uri_free (uri); } - if (bytes_sig && !_ostree_repo_load_cache_summary_if_same_sig (self, - remote_name_or_baseurl, - bytes_sig, - &bytes_summary, - cancellable, - error)) + if (bytes_sig && + !_ostree_repo_remote_name_is_file (remote_name_or_baseurl) && + !_ostree_repo_load_cache_summary_if_same_sig (self, + remote_name_or_baseurl, + bytes_sig, + &bytes_summary, + cancellable, + error)) goto out; if (bytes_summary) @@ -2160,7 +2162,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!summary_from_cache && bytes_summary && bytes_sig) { - if (!_ostree_repo_cache_summary (self, + if (!_ostree_repo_remote_name_is_file (remote_name_or_baseurl) && + !_ostree_repo_cache_summary (self, remote_name_or_baseurl, bytes_summary, bytes_sig, From 87298c84b81a76ad4858d9bb6d60d25cffa4e7c5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 4 Apr 2016 14:43:33 +0200 Subject: [PATCH 37/69] Fix local-pull test This was not being run for whatever reason. Also, the test count was wrong. Closes: #237 Approved by: cgwalters --- Makefile-tests.am | 2 +- tests/test-local-pull.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index b7ff268d..4bf12ce8 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -50,6 +50,7 @@ test_scripts = \ tests/test-pull-summary-sigs.sh \ tests/test-pull-resume.sh \ tests/test-pull-untrusted.sh \ + tests/test-local-pull.sh \ tests/test-local-pull-depth.sh \ tests/test-gpg-signed-commit.sh \ tests/test-admin-upgrade-unconfigured.sh \ @@ -92,7 +93,6 @@ installed_test_data = tests/archive-test.sh \ tests/admin-test.sh \ tests/basic-test.sh \ tests/test-basic-user.sh \ - tests/test-local-pull.sh \ tests/corrupt-repo-ref.js \ tests/pre-endian-deltas-repo-big.tar.xz \ tests/pre-endian-deltas-repo-little.tar.xz \ diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index 50ecbcb7..f158f962 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -23,7 +23,7 @@ set -euo pipefail skip_without_user_xattrs -echo "1..1" +echo "1..4" setup_test_repository "archive-z2" echo "ok setup" From ace0d4650b097b8d3fb04cc729715a92f6db7dee Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 4 Apr 2016 15:17:17 +0200 Subject: [PATCH 38/69] pull-local: Support --gpg-verify and --gpg-verify-summary Force the otherwise disabled gpg verifications on. Note: You need to pass --remote=foo so we know what gpg keys to verify against. Closes: #237 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 20 ++++++++++++--- src/ostree/ot-builtin-pull-local.c | 12 ++++++++- tests/test-local-pull.sh | 40 +++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 238fdf65..4d8b6138 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1900,6 +1900,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; gboolean require_static_deltas = FALSE; + gboolean opt_gpg_verify = FALSE; + gboolean opt_gpg_verify_summary = FALSE; if (options) { @@ -1910,6 +1912,8 @@ ostree_repo_pull_with_options (OstreeRepo *self, flags = flags_i; (void) g_variant_lookup (options, "subdir", "&s", &dir_to_pull); (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); + (void) g_variant_lookup (options, "gpg-verify", "b", &opt_gpg_verify); + (void) g_variant_lookup (options, "gpg-verify-summary", "b", &opt_gpg_verify_summary); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); @@ -1967,10 +1971,18 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (_ostree_repo_remote_name_is_file (remote_name_or_baseurl)) { /* For compatibility with pull-local, don't gpg verify local - * pulls. + * pulls by default. */ - pull_data->gpg_verify = FALSE; - pull_data->gpg_verify_summary = FALSE; + pull_data->gpg_verify = opt_gpg_verify; + pull_data->gpg_verify_summary = opt_gpg_verify_summary; + + if ((pull_data->gpg_verify || pull_data->gpg_verify_summary) && + pull_data->remote_name == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Must specify remote name to enable gpg verification"); + goto out; + } } else { @@ -2181,7 +2193,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, result = _ostree_repo_gpg_verify_with_metadata (self, bytes_summary, sig_variant, - remote_name_or_baseurl, + pull_data->remote_name, NULL, NULL, cancellable, diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c index f3ca184a..36057ec6 100644 --- a/src/ostree/ot-builtin-pull-local.c +++ b/src/ostree/ot-builtin-pull-local.c @@ -33,12 +33,16 @@ static char *opt_remote; static gboolean opt_disable_fsync; static gboolean opt_untrusted; +static gboolean opt_gpg_verify; +static gboolean opt_gpg_verify_summary; static int opt_depth = 0; static GOptionEntry options[] = { { "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "untrusted", 0, 0, G_OPTION_ARG_NONE, &opt_untrusted, "Do not trust source", NULL }, + { "gpg-verify", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify, "GPG verify commits (must specify --remote)", NULL }, + { "gpg-verify-summary", 0, 0, G_OPTION_ARG_NONE, &opt_gpg_verify_summary, "GPG verify summary (must specify --remote)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -145,9 +149,15 @@ ostree_builtin_pull_local (int argc, char **argv, GCancellable *cancellable, GEr if (opt_remote) g_variant_builder_add (&builder, "{s@v}", "override-remote-name", g_variant_new_variant (g_variant_new_string (opt_remote))); + if (opt_gpg_verify) + g_variant_builder_add (&builder, "{s@v}", "gpg-verify", + g_variant_new_variant (g_variant_new_boolean (TRUE))); + if (opt_gpg_verify_summary) + g_variant_builder_add (&builder, "{s@v}", "gpg-verify-summary", + g_variant_new_variant (g_variant_new_boolean (TRUE))); g_variant_builder_add (&builder, "{s@v}", "depth", g_variant_new_variant (g_variant_new_int32 (opt_depth))); - + if (!ostree_repo_pull_with_options (repo, src_repo_uri, g_variant_builder_end (&builder), progress, diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index f158f962..a9ac1278 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -19,11 +19,14 @@ set -euo pipefail +# We don't want OSTREE_GPG_HOME used for these tests. +unset OSTREE_GPG_HOME + . $(dirname $0)/libtest.sh skip_without_user_xattrs -echo "1..4" +echo "1..7" setup_test_repository "archive-z2" echo "ok setup" @@ -57,3 +60,38 @@ find checkout3 -printf '%P %s %#m %u/%g %y %l\n' | sort > checkout3.files cmp checkout1.files checkout2.files cmp checkout1.files checkout3.files echo "ok checkouts same" + +mkdir repo4 +${CMD_PREFIX} ostree --repo=repo4 init --mode="archive-z2" +${CMD_PREFIX} ostree --repo=repo4 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo +if ${CMD_PREFIX} ostree --repo=repo4 pull-local --remote=origin --gpg-verify repo test2 2>&1; then + assert_not_reached "GPG verification unexpectedly succeeded" +fi +echo "ok --gpg-verify with no signature" + +${OSTREE} gpg-sign --gpg-homedir=${TEST_GPG_KEYHOME} test2 ${TEST_GPG_KEYID_1} + +mkdir repo5 +${CMD_PREFIX} ostree --repo=repo5 init --mode="archive-z2" +${CMD_PREFIX} ostree --repo=repo5 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo +${CMD_PREFIX} ostree --repo=repo5 pull-local --remote=origin --gpg-verify repo test2 +echo "ok --gpg-verify" + +mkdir repo6 +${CMD_PREFIX} ostree --repo=repo6 init --mode="archive-z2" +${CMD_PREFIX} ostree --repo=repo6 remote add --gpg-import ${test_tmpdir}/gpghome/key1.asc origin repo +if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then + assert_not_reached "GPG summary verification with no summary unexpectedly succeeded" +fi + +${OSTREE} summary -u update + +if ${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1; then + assert_not_reached "GPG summary verification with signed no summary unexpectedly succeeded" +fi + +${OSTREE} summary -u update --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} + +${CMD_PREFIX} ostree --repo=repo6 pull-local --remote=origin --gpg-verify-summary repo test2 2>&1 + +echo "ok --gpg-verify-summary" From e9640ca4d91fffa899f7a7ba14bae798215fb6a6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 Apr 2016 14:22:19 -0400 Subject: [PATCH 39/69] build: Find grub2-mkconfig a bit more automagically If one happens to not have grub2 installed, the previous commit would cause us to fall back to `grub-mkconfig` with bad results. We should likely just do dynamic detection and avoid pain, but there's no harm in also allowing this to be statically determined. The automagic here is that if `/etc/os-release` says `ID_LIKE=fedora`, we know to use `grub2-mkconfig`. But distro scripts should set this in the build configurations using `--with-grub2-mkconfig=`. Closes: #240 Approved by: jlebon --- configure.ac | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index 7dfcd500..24dfb356 100644 --- a/configure.ac +++ b/configure.ac @@ -257,14 +257,17 @@ AC_ARG_WITH(builtin-grub2-mkconfig, [with_builtin_grub2_mkconfig=no]) AM_CONDITIONAL(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, test x$with_builtin_grub2_mkconfig = xyes) AM_COND_IF(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, - AC_DEFINE([USE_BUILTIN_GRUB2_MKCONFIG], 1, [Define if using internal ostree-grub-generator]), -[ - # Otherwise, look for the path to the system generator. On some - # distributions GRUB2 *-mkconfig executable has 'grub2' prefix and - # on some 'grub'. - AC_CHECK_PROG(GRUB2_MKCONFIG, grub2-mkconfig, grub2-mkconfig, grub-mkconfig) - AC_DEFINE_UNQUOTED([GRUB2_MKCONFIG_PATH], ["$GRUB2_MKCONFIG"], [The system grub2-mkconfig executible name]) -]) + AC_DEFINE([USE_BUILTIN_GRUB2_MKCONFIG], 1, [Define if using internal ostree-grub-generator])) +AC_ARG_WITH(grub2-mkconfig-path, + AS_HELP_STRING([--with-grub2-mkconfig-path], + [Path to grub2-mkconfig])) +AS_IF([test x$with_grub2_mkconfig_path = x], [ + dnl Otherwise, look for the path to the system generator. On some + dnl distributions GRUB2 *-mkconfig executable has 'grub2' prefix and + dnl on some 'grub'. We default to grub2-mkconfig. + AC_CHECK_PROGS(GRUB2_MKCONFIG, [grub2-mkconfig grub-mkconfig], [grub2-mkconfig]) +],[GRUB2_MKCONFIG=$with_grub2_mkconfig_path]) +AC_DEFINE_UNQUOTED([GRUB2_MKCONFIG_PATH], ["$GRUB2_MKCONFIG"], [The system grub2-mkconfig executible name]) dnl for tests AS_IF([test "x$found_introspection" = xyes], [ @@ -301,8 +304,12 @@ echo " api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs dracut: $with_dracut - mkinitcpio: $with_mkinitcpio - builtin grub2-mkconfig (instead of system): $with_builtin_grub2_mkconfig" + mkinitcpio: $with_mkinitcpio" +AS_IF([test x$with_builtin_grub2_mkconfig = xyes], [ + echo " builtin grub2-mkconfig (instead of system): $with_builtin_grub2_mkconfig" +], [ + echo " grub2-mkconfig path: $GRUB2_MKCONFIG" +]) AS_IF([test "x$with_systemd" = "xyes"], [ echo " systemd unit dir: $with_systemdsystemunitdir" ]) From 4b1ac83aa6f8d13310163cadd15c7ee3b26ae7ae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 6 Apr 2016 21:26:38 -0400 Subject: [PATCH 40/69] build: Make tests/libreaddir-rand.so rule use AM_V_GEN So non-verbose builds don't have a verbose rule smack in the middle. Closes: #241 Approved by: giuseppe --- Makefile-tests.am | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile-tests.am b/Makefile-tests.am index 4bf12ce8..c8e201ba 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -222,9 +222,9 @@ EXTRA_DIST += \ tests/gpg-verify-data/trustdb.gpg \ tests/gpg-verify-data/gpg.conf -tests-libreaddir-rand-so-symlink: - ln -fns ../.libs/libreaddir-rand.so tests -ALL_LOCAL_RULES += tests-libreaddir-rand-so-symlink +tests/libreaddir-rand.so: Makefile + $(AM_V_GEN) ln -fns ../.libs/libreaddir-rand.so tests +ALL_LOCAL_RULES += tests/libreaddir-rand.so # Unfortunately the glib test data APIs don't actually handle # non-recursive Automake, so we change our code to canonically look From 039d3f1dbd1dcf28ccf4e08b46f8692155af4c1f Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 7 Apr 2016 09:53:54 +0200 Subject: [PATCH 41/69] tests: add libostreetest.h to EXTRA_DIST Signed-off-by: Giuseppe Scrivano Closes: #242 Approved by: cgwalters --- Makefile-tests.am | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile-tests.am b/Makefile-tests.am index c8e201ba..11b881a3 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -214,6 +214,7 @@ tests_test_gpg_verify_result_CFLAGS = $(TESTS_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS tests_test_gpg_verify_result_LDADD = $(TESTS_LDADD) $(OT_INTERNAL_GPGME_LIBS) EXTRA_DIST += \ + tests/libostreetest.h \ tests/gpg-verify-data/README.md \ tests/gpg-verify-data/lgpl2 \ tests/gpg-verify-data/lgpl2.sig \ From 7763c452dd88dd91c25b41b25e94896e68d11794 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 7 Apr 2016 09:55:54 +0200 Subject: [PATCH 42/69] tests: add missing ${CMD_PREFIX} before ostree Signed-off-by: Giuseppe Scrivano Closes: #242 Approved by: cgwalters --- tests/test-demo-buildsystem.sh | 18 +++++++++--------- tests/test-prune.sh | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test-demo-buildsystem.sh b/tests/test-demo-buildsystem.sh index 97915f94..cc926979 100755 --- a/tests/test-demo-buildsystem.sh +++ b/tests/test-demo-buildsystem.sh @@ -42,14 +42,14 @@ exampleos_build_commit_package() { mkdir -p ${pkg}-package/usr/bin/ echo "${pkg}-content ${version}" > ${pkg}-package/usr/bin/${pkg} # Use a dummy subject for this. - ostree --repo=build-repo commit -b exampleos/x86_64/${pkg} -s '' --tree=dir=${pkg}-package + ${CMD_PREFIX} ostree --repo=build-repo commit -b exampleos/x86_64/${pkg} -s '' --tree=dir=${pkg}-package rm ${pkg}-package -rf } exampleos_recompose() { rm exampleos-build -rf for pkg in ${packages}; do - ostree --repo=build-repo checkout -U --union exampleos/x86_64/${pkg} exampleos-build + ${CMD_PREFIX} ostree --repo=build-repo checkout -U --union exampleos/x86_64/${pkg} exampleos-build done # Now that we have our rootfs, run triggers @@ -60,15 +60,15 @@ exampleos_recompose() { # Then we commit it, using --link-checkout-speedup to effectively # only re-checksum the ldconfig file. We also have dummy commit # message here. - ostree --repo=build-repo commit -b exampleos/x86_64/standard -s 'exampleos build' --link-checkout-speedup exampleos-build + ${CMD_PREFIX} ostree --repo=build-repo commit -b exampleos/x86_64/standard -s 'exampleos build' --link-checkout-speedup exampleos-build } packages="bash systemd" mkdir build-repo -ostree --repo=build-repo init --mode=bare-user +${CMD_PREFIX} ostree --repo=build-repo init --mode=bare-user mkdir repo -ostree --repo=repo init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=repo init --mode=archive-z2 # Our FUSE mount point mkdir mnt @@ -81,7 +81,7 @@ exampleos_build_commit_package systemd 224 exampleos_recompose # This is our first commit - let's publish it. -ostree --repo=repo pull-local build-repo exampleos/x86_64/standard +${CMD_PREFIX} ostree --repo=repo pull-local build-repo exampleos/x86_64/standard # Now, update the bash package - this is a new commit on the branch # exampleos/x86_64/bash. @@ -91,11 +91,11 @@ exampleos_build_commit_package bash 0.5.0 exampleos_recompose # Publish again: -ostree --repo=repo pull-local build-repo exampleos/x86_64/standard +${CMD_PREFIX} ostree --repo=repo pull-local build-repo exampleos/x86_64/standard # Optional: Generate a static delta vs the previous build -ostree --repo=repo static-delta generate exampleos/x86_64/standard +${CMD_PREFIX} ostree --repo=repo static-delta generate exampleos/x86_64/standard # Optional: Regenerate the summary file -ostree --repo=repo summary -u +${CMD_PREFIX} ostree --repo=repo summary -u # Try: ostree --repo=demo-repo ls -R exampleos/x86_64/standard diff --git a/tests/test-prune.sh b/tests/test-prune.sh index e1796a33..d32edbc0 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -130,9 +130,9 @@ assert_file_has_content deltascount "^1$" echo "ok prune" rm repo -rf -ostree --repo=repo init --mode=bare-user +${CMD_PREFIX} ostree --repo=repo init --mode=bare-user ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 --commit-metadata-only origin test -ostree --repo=repo prune +${CMD_PREFIX} ostree --repo=repo prune echo "ok prune with partial repo" From 6a091c96ba7ea5dcbc83a4c601d4b8d33e4874f0 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 7 Apr 2016 10:11:11 +0200 Subject: [PATCH 43/69] contrib: indent golang code using only tabs instead of both tabs and spaces Signed-off-by: Giuseppe Scrivano Closes: #242 Approved by: cgwalters --- contrib/golang/glibobject.go | 66 +++++++++++++-------------- contrib/golang/ostree.go | 86 +++++++++++++++++------------------ contrib/golang/ostree_test.go | 74 +++++++++++++++--------------- 3 files changed, 113 insertions(+), 113 deletions(-) diff --git a/contrib/golang/glibobject.go b/contrib/golang/glibobject.go index 585ccd70..0b62b443 100644 --- a/contrib/golang/glibobject.go +++ b/contrib/golang/glibobject.go @@ -47,50 +47,50 @@ func GoBool(b C.gboolean) bool { } type GError struct { - ptr unsafe.Pointer + ptr unsafe.Pointer } func NewGError() GError { - return GError{nil} + return GError{nil} } func (e *GError) Native() *C.GError { - if e == nil { - return nil - } - return (*C.GError)(e.ptr) + if e == nil { + return nil + } + return (*C.GError)(e.ptr) } func ConvertGError(e *C.GError) error { - defer C.g_error_free(e) - return errors.New(C.GoString((*C.char)(C._g_error_get_message(e)))) + defer C.g_error_free(e) + return errors.New(C.GoString((*C.char)(C._g_error_get_message(e)))) } type GType uint func (t GType) Name() string { - return C.GoString((*C.char)(C.g_type_name(C.GType(t)))) + return C.GoString((*C.char)(C.g_type_name(C.GType(t)))) } - + type GVariant struct { - ptr unsafe.Pointer + ptr unsafe.Pointer } func GVariantNew(p unsafe.Pointer) *GVariant { - o := &GVariant{p} - runtime.SetFinalizer(o, (*GVariant).Unref) - return o; + o := &GVariant{p} + runtime.SetFinalizer(o, (*GVariant).Unref) + return o; } func GVariantNewSink(p unsafe.Pointer) *GVariant { - o := &GVariant{p} - runtime.SetFinalizer(o, (*GVariant).Unref) - o.RefSink() - return o; + o := &GVariant{p} + runtime.SetFinalizer(o, (*GVariant).Unref) + o.RefSink() + return o; } func (v *GVariant) native() *C.GVariant { - return (*C.GVariant)(v.ptr); + return (*C.GVariant)(v.ptr); } func (v *GVariant) Ref() { @@ -98,7 +98,7 @@ func (v *GVariant) Ref() { } func (v *GVariant) Unref() { - C.g_variant_unref(v.native()) + C.g_variant_unref(v.native()) } func (v *GVariant) RefSink() { @@ -116,14 +116,14 @@ func (v *GVariant) GetChildValue(i int) *GVariant { } func (v *GVariant) LookupString(key string) (string, error) { - ckey := C.CString(key) - defer C.free(unsafe.Pointer(ckey)) - // TODO: Find a way to have constant C strings in golang - cstr := C._g_variant_lookup_string(v.native(), ckey) - if cstr == nil { - return "", fmt.Errorf("No such key: %s", key) - } - return C.GoString(cstr), nil + ckey := C.CString(key) + defer C.free(unsafe.Pointer(ckey)) + // TODO: Find a way to have constant C strings in golang + cstr := C._g_variant_lookup_string(v.native(), ckey) + if cstr == nil { + return "", fmt.Errorf("No such key: %s", key) + } + return C.GoString(cstr), nil } /* @@ -144,9 +144,9 @@ type GObject struct { } func GObjectNew(p unsafe.Pointer) *GObject { - o := &GObject{p} - runtime.SetFinalizer(o, (*GObject).Unref) - return o; + o := &GObject{p} + runtime.SetFinalizer(o, (*GObject).Unref) + return o; } func (v *GObject) Ptr() unsafe.Pointer { @@ -172,7 +172,7 @@ func (v *GObject) Ref() { } func (v *GObject) Unref() { - C.g_object_unref(C.gpointer(v.ptr)) + C.g_object_unref(C.gpointer(v.ptr)) } func (v *GObject) RefSink() { @@ -191,7 +191,7 @@ func (v *GObject) ForceFloating() { // GIO types type GCancellable struct { - *GObject + *GObject } func (self *GCancellable) native() *C.GCancellable { diff --git a/contrib/golang/ostree.go b/contrib/golang/ostree.go index 0a60ef68..8891319c 100644 --- a/contrib/golang/ostree.go +++ b/contrib/golang/ostree.go @@ -16,11 +16,11 @@ import ( import "C" type Repo struct { - *GObject + *GObject } func RepoGetType() GType { - return GType(C.ostree_repo_get_type()) + return GType(C.ostree_repo_get_type()) } func (r *Repo) native() *C.OstreeRepo { @@ -28,67 +28,67 @@ func (r *Repo) native() *C.OstreeRepo { } func repoFromNative(p *C.OstreeRepo) *Repo { - if p == nil { - return nil - } - o := GObjectNew(unsafe.Pointer(p)) - r := &Repo{o} - return r + if p == nil { + return nil + } + o := GObjectNew(unsafe.Pointer(p)) + r := &Repo{o} + return r } func RepoNewOpen(path string) (*Repo, error) { - var cerr *C.GError = nil - cpath := C.CString(path) - pathc := C.g_file_new_for_path(cpath); - defer C.g_object_unref(C.gpointer(pathc)) - crepo := C.ostree_repo_new(pathc) - repo := repoFromNative(crepo); - r := GoBool(C.ostree_repo_open(repo.native(), nil, &cerr)) - if !r { - return nil, ConvertGError(cerr) - } - return repo, nil + var cerr *C.GError = nil + cpath := C.CString(path) + pathc := C.g_file_new_for_path(cpath); + defer C.g_object_unref(C.gpointer(pathc)) + crepo := C.ostree_repo_new(pathc) + repo := repoFromNative(crepo); + r := GoBool(C.ostree_repo_open(repo.native(), nil, &cerr)) + if !r { + return nil, ConvertGError(cerr) + } + return repo, nil } func (r *Repo) GetParent() *Repo { - return repoFromNative(C.ostree_repo_get_parent(r.native())) + return repoFromNative(C.ostree_repo_get_parent(r.native())) } type ObjectType int const ( - OBJECT_TYPE_FILE ObjectType = C.OSTREE_OBJECT_TYPE_FILE - OBJECT_TYPE_DIR_TREE = C.OSTREE_OBJECT_TYPE_DIR_TREE - OBJECT_TYPE_DIR_META = C.OSTREE_OBJECT_TYPE_DIR_META - OBJECT_TYPE_COMMIT = C.OSTREE_OBJECT_TYPE_COMMIT - OBJECT_TYPE_TOMBSTONE_COMMIT = C.OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT + OBJECT_TYPE_FILE ObjectType = C.OSTREE_OBJECT_TYPE_FILE + OBJECT_TYPE_DIR_TREE = C.OSTREE_OBJECT_TYPE_DIR_TREE + OBJECT_TYPE_DIR_META = C.OSTREE_OBJECT_TYPE_DIR_META + OBJECT_TYPE_COMMIT = C.OSTREE_OBJECT_TYPE_COMMIT + OBJECT_TYPE_TOMBSTONE_COMMIT = C.OSTREE_OBJECT_TYPE_TOMBSTONE_COMMIT ) func (repo *Repo) LoadVariant(t ObjectType, checksum string) (*GVariant, error) { - var cerr *C.GError = nil - var cvariant *C.GVariant = nil + var cerr *C.GError = nil + var cvariant *C.GVariant = nil - r := GoBool(C.ostree_repo_load_variant(repo.native(), C.OstreeObjectType(t), C.CString(checksum), &cvariant, &cerr)) - if !r { - return nil, ConvertGError(cerr) - } - variant := GVariantNew(unsafe.Pointer(cvariant)) - return variant, nil + r := GoBool(C.ostree_repo_load_variant(repo.native(), C.OstreeObjectType(t), C.CString(checksum), &cvariant, &cerr)) + if !r { + return nil, ConvertGError(cerr) + } + variant := GVariantNew(unsafe.Pointer(cvariant)) + return variant, nil } func (repo *Repo) ResolveRev(ref string) (string, error) { - var cerr *C.GError = nil - var crev *C.char = nil + var cerr *C.GError = nil + var crev *C.char = nil - r := GoBool(C.ostree_repo_resolve_rev(repo.native(), C.CString(ref), GBool(true), &crev, &cerr)) - if !r { - return "", ConvertGError(cerr) - } - defer C.free(unsafe.Pointer(crev)) - return C.GoString(crev), nil + r := GoBool(C.ostree_repo_resolve_rev(repo.native(), C.CString(ref), GBool(true), &crev, &cerr)) + if !r { + return "", ConvertGError(cerr) + } + defer C.free(unsafe.Pointer(crev)) + return C.GoString(crev), nil } func (commit *GVariant) CommitGetMetadataKeyString(key string) (string, error) { - cmeta := GVariantNew(unsafe.Pointer(C.g_variant_get_child_value(commit.native(), 0))) - return cmeta.LookupString(key) + cmeta := GVariantNew(unsafe.Pointer(C.g_variant_get_child_value(commit.native(), 0))) + return cmeta.LookupString(key) } diff --git a/contrib/golang/ostree_test.go b/contrib/golang/ostree_test.go index 6b687d8b..0ffcb858 100644 --- a/contrib/golang/ostree_test.go +++ b/contrib/golang/ostree_test.go @@ -9,47 +9,47 @@ import ( ) func TestTypeName(t *testing.T) { - name := RepoGetType().Name(); - if name != "OstreeRepo" { - t.Errorf("%s != OstreeRepo"); - } + name := RepoGetType().Name(); + if name != "OstreeRepo" { + t.Errorf("%s != OstreeRepo"); + } } func TestRepoNew(t *testing.T) { - r, err := RepoNewOpen("/ostree/repo") - if err != nil { - t.Errorf("%s", err); - return - } - parent := r.GetParent() - if parent != nil { - t.Errorf("Expected no parent") - return - } + r, err := RepoNewOpen("/ostree/repo") + if err != nil { + t.Errorf("%s", err); + return + } + parent := r.GetParent() + if parent != nil { + t.Errorf("Expected no parent") + return + } } func TestRepoGetMetadataVersion(t *testing.T) { - r, err := RepoNewOpen("/ostree/repo") - if err != nil { - t.Errorf("%s", err); - return - } - commit,err := r.ResolveRev("rhel-atomic-host/7/x86_64/standard") - if err != nil { - t.Errorf("%s", err) - return - } - commitv,err := r.LoadVariant(OBJECT_TYPE_COMMIT, commit) - if err != nil { - t.Errorf("%s", err) - return - } - ver, err := commitv.CommitGetMetadataKeyString("version") - if err != nil { - t.Errorf("%s", err) - return - } - if ver != "7.1.3" { - t.Errorf("expected 7.1.3") - } + r, err := RepoNewOpen("/ostree/repo") + if err != nil { + t.Errorf("%s", err); + return + } + commit,err := r.ResolveRev("rhel-atomic-host/7/x86_64/standard") + if err != nil { + t.Errorf("%s", err) + return + } + commitv,err := r.LoadVariant(OBJECT_TYPE_COMMIT, commit) + if err != nil { + t.Errorf("%s", err) + return + } + ver, err := commitv.CommitGetMetadataKeyString("version") + if err != nil { + t.Errorf("%s", err) + return + } + if ver != "7.1.3" { + t.Errorf("expected 7.1.3") + } } From 3152516ab326bcc6f070cecda7c4c8645a97df19 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 7 Apr 2016 10:06:54 +0200 Subject: [PATCH 44/69] Remove empty new lines at the EOF Signed-off-by: Giuseppe Scrivano Closes: #242 Approved by: cgwalters --- .travis.yml | 1 - docs/manual/repository-management.md | 3 --- src/libostree/ostree-sysroot.c | 1 - src/switchroot/ostree-mount-util.c | 1 - 4 files changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a80c942..69ea10fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,4 +22,3 @@ notifications: branches: only: - auto - diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md index df92f6b9..ceb96cad 100644 --- a/docs/manual/repository-management.md +++ b/docs/manual/repository-management.md @@ -209,6 +209,3 @@ That will truncate the history older than 6 months. Deleted commits will have "tombstone markers" added so that you know they were explicitly deleted, but all content in them (that is not referenced by a still retained commit) will be garbage collected. - - - diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 6ee3dff9..b114a901 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -1850,4 +1850,3 @@ ostree_sysroot_deployment_unlock (OstreeSysroot *self, out: return ret; } - diff --git a/src/switchroot/ostree-mount-util.c b/src/switchroot/ostree-mount-util.c index daec66c5..bb27026c 100644 --- a/src/switchroot/ostree-mount-util.c +++ b/src/switchroot/ostree-mount-util.c @@ -66,4 +66,3 @@ path_is_on_readonly_fs (char *path) return (stvfsbuf.f_flag & ST_RDONLY) != 0; } - From e81aabba8c4e7c1e773b566f7229d5a3a3d82619 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Thu, 7 Apr 2016 10:13:45 +0200 Subject: [PATCH 45/69] docs: Prefer the form "cannot" to "can not" Signed-off-by: Giuseppe Scrivano Closes: #242 Approved by: cgwalters --- docs/manual/related-projects.md | 2 +- docs/manual/repository-management.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index b53ba792..73972330 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -63,7 +63,7 @@ awareness of BTRFS in dpkg/rpm itself) will be required. The OSTree author believes that having total freedom at the block storage layer is better for general purpose operating systems. For example, with OSTree, one is free to use BTRFS in any way you like - -you can use a subvolume for `/home`, or you can not. +you can use a subvolume for `/home`, or you cannot. Furthermore, in its most basic incarnation, the rpm/dpkg + BTRFS doesn't solve the race conditions that happen when unpacking packages diff --git a/docs/manual/repository-management.md b/docs/manual/repository-management.md index ceb96cad..b83f6c15 100644 --- a/docs/manual/repository-management.md +++ b/docs/manual/repository-management.md @@ -179,7 +179,7 @@ you should update the summary file: ostree --repo=repo-prod summary -u ``` -(Remember, the `summary` command can not be run concurrently, so this +(Remember, the `summary` command cannot be run concurrently, so this should be triggered serially by other jobs). There is some more information on the design of the summary file in From 984bf91826187140682947fb8e0a2cc0a8a3f355 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 31 Mar 2016 15:08:45 -0400 Subject: [PATCH 46/69] Use git.mk It's a lot nicer than manually maintaining .gitignore in general. Closes: #235 Approved by: giuseppe --- .gitignore | 99 ------------- Makefile-decls.am | 2 - Makefile-tests.am | 1 + Makefile.am | 4 + apidoc/.gitignore | 53 ++++++- apidoc/Makefile.am | 2 + git.mk | 348 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 406 insertions(+), 103 deletions(-) delete mode 100644 .gitignore create mode 100644 git.mk diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 0eb6981b..00000000 --- a/.gitignore +++ /dev/null @@ -1,99 +0,0 @@ -# Standard C/Automake goo -.deps -.libs -.dirstamp -*.typelib -*.la -*.lo -*.o -*.pyc -*.stamp -*~ -Makefile -Makefile.in -aclocal.m4 -autom4te.cache -compile -config.guess -config.h -config.h.in -config.log -config.status -config.sub -configure -depcomp -gtk-doc.make -INSTALL -install-sh -libtool -ltmain.sh -missing -stamp-h1 -ylwrap -py-compile -config -m4 -po -ABOUT-NLS -_build - -/build-aux/test-driver -/buildutil/gtk-doc.m4 -/buildutil/libtool.m4 -/buildutil/ltoptions.m4 -/buildutil/ltsugar.m4 -/buildutil/ltversion.m4 -/buildutil/lt~obsolete.m4 - -/doc/ostree-decl.txt -/doc/ostree-overrides.txt -/doc/ostree-undeclared.txt -/doc/ostree-undocumented.txt -/doc/ostree-unused.txt -/doc/ostree.types -/doc/version.xml -/doc/tmpl -/doc/html -/doc/xml -/doc/ostree.1 - -/man/*.1 -/man/*.5 - -/ostree -/ostree-prepare-root -/ostree-remount -/OSTree-1.0.gir - -/rofiles-fuse - -/src/libostree/ostree-1.pc -/src/libostree/ostree-enumtypes.c -/src/libostree/ostree-enumtypes.h -/src/ostree/parse-datetime.c - -/test-admin-deploy-1.test -/test-admin-deploy-2.test -/test-archivez.test -/test-basic.test -/test-corruption.test -/test-libarchive.test -/test-pull-archive-z.test -/test-pull-corruption.test -/test-pull-resume.test -/test-remote-add.test -/test-setuid.test -/test-xattrs.test -/tests/libreaddir-rand.so -/tests/test-basic-c -/tests/test-libarchive-import -/tests/test-lzma -/tests/test-sysroot-c -test-varint -test*.test -*.trs -*.log - -*.tar.xz -*.src.rpm -packaging/*.spec diff --git a/Makefile-decls.am b/Makefile-decls.am index eefffd64..d8ec5ab4 100644 --- a/Makefile-decls.am +++ b/Makefile-decls.am @@ -43,8 +43,6 @@ gir_DATA = typelibdir = $(libdir)/girepository-1.0 typelib_DATA = gsettings_SCHEMAS = -# git.mk -GITIGNOREFILES = # This is a special facility to chain together hooks easily INSTALL_DATA_HOOKS = diff --git a/Makefile-tests.am b/Makefile-tests.am index 11b881a3..4adfbf7b 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -226,6 +226,7 @@ EXTRA_DIST += \ tests/libreaddir-rand.so: Makefile $(AM_V_GEN) ln -fns ../.libs/libreaddir-rand.so tests ALL_LOCAL_RULES += tests/libreaddir-rand.so +CLEANFILES += tests/libreaddir-rand.so # Unfortunately the glib test data APIs don't actually handle # non-recursive Automake, so we change our code to canonically look diff --git a/Makefile.am b/Makefile.am index e25842d9..488d4b6d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -30,6 +30,8 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ AM_CFLAGS += $(WARN_CFLAGS) DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode +GITIGNOREFILES = aclocal.m4 build-aux/ buildutil/*.m4 config.h.in gtk-doc.make + SUBDIRS += . if ENABLE_GTK_DOC @@ -101,3 +103,5 @@ release-tarball-embedded: $(embed_dependency) embedded-dependencies/libsoup; \ mv ostree-embeddeps-$${GITVERSION}.tar{.tmp,}; \ gzip -f ostree-embeddeps-$${GITVERSION}.tar + +-include $(top_srcdir)/git.mk diff --git a/apidoc/.gitignore b/apidoc/.gitignore index e0b4a4a6..77cacd4d 100644 --- a/apidoc/.gitignore +++ b/apidoc/.gitignore @@ -1,2 +1,51 @@ -ostree*.1 -ostree*.5 +*.lo +*.o +.deps +.libs +/*.bak +/*.gcda +/*.gcno +/*.orig +/*.rej +/*.tab.c +/*~ +/.*.sw[nop] +/.dirstamp +/.gitignore +/GPATH +/GRTAGS +/GSYMS +/GTAGS +/ID +/Makefile +/Makefile.in +/TAGS +/gtkdoc-check.test +/html +/html-build.stamp +/html.stamp +/ostree-decl-list.txt +/ostree-decl.txt +/ostree-undeclared.txt +/ostree-undocumented.txt +/ostree-unused.txt +/ostree.args +/ostree.hierarchy +/ostree.interfaces +/ostree.pdf +/ostree.prerequisites +/ostree.signals +/ostree.types +/pdf-build.stamp +/pdf.stamp +/scan-build.stamp +/setup-build.stamp +/sgml-build.stamp +/sgml.stamp +/so_locations +/tags +/tmpl +/tmpl/*.bak +/tmpl/ostree-unused.sgml +/xml +_libs diff --git a/apidoc/Makefile.am b/apidoc/Makefile.am index dcb009ef..baa7207a 100644 --- a/apidoc/Makefile.am +++ b/apidoc/Makefile.am @@ -121,3 +121,5 @@ EXTRA_DIST += \ version.xml \ ostree-sections.txt \ $(NULL) + +-include $(top_srcdir)/git.mk diff --git a/git.mk b/git.mk new file mode 100644 index 00000000..643c2ca9 --- /dev/null +++ b/git.mk @@ -0,0 +1,348 @@ +# git.mk, a small Makefile to autogenerate .gitignore files +# for autotools-based projects. +# +# Copyright 2009, Red Hat, Inc. +# Copyright 2010,2011,2012,2013 Behdad Esfahbod +# Written by Behdad Esfahbod +# +# Copying and distribution of this file, with or without modification, +# is permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# The latest version of this file can be downloaded from: +GIT_MK_URL = https://raw.githubusercontent.com/behdad/git.mk/master/git.mk +# +# Bugs, etc, should be reported upstream at: +# https://github.com/behdad/git.mk +# +# To use in your project, import this file in your git repo's toplevel, +# then do "make -f git.mk". This modifies all Makefile.am files in +# your project to -include git.mk. Remember to add that line to new +# Makefile.am files you create in your project, or just rerun the +# "make -f git.mk". +# +# This enables automatic .gitignore generation. If you need to ignore +# more files, add them to the GITIGNOREFILES variable in your Makefile.am. +# But think twice before doing that. If a file has to be in .gitignore, +# chances are very high that it's a generated file and should be in one +# of MOSTLYCLEANFILES, CLEANFILES, DISTCLEANFILES, or MAINTAINERCLEANFILES. +# +# The only case that you need to manually add a file to GITIGNOREFILES is +# when remove files in one of mostlyclean-local, clean-local, distclean-local, +# or maintainer-clean-local make targets. +# +# Note that for files like editor backup, etc, there are better places to +# ignore them. See "man gitignore". +# +# If "make maintainer-clean" removes the files but they are not recognized +# by this script (that is, if "git status" shows untracked files still), send +# me the output of "git status" as well as your Makefile.am and Makefile for +# the directories involved and I'll diagnose. +# +# For a list of toplevel files that should be in MAINTAINERCLEANFILES, see +# Makefile.am.sample in the git.mk git repo. +# +# Don't EXTRA_DIST this file. It is supposed to only live in git clones, +# not tarballs. It serves no useful purpose in tarballs and clutters the +# build dir. +# +# This file knows how to handle autoconf, automake, libtool, gtk-doc, +# gnome-doc-utils, yelp.m4, mallard, intltool, gsettings, dejagnu, appdata, +# appstream. +# +# This makefile provides the following targets: +# +# - all: "make all" will build all gitignore files. +# - gitignore: makes all gitignore files in the current dir and subdirs. +# - .gitignore: make gitignore file for the current dir. +# - gitignore-recurse: makes all gitignore files in the subdirs. +# +# KNOWN ISSUES: +# +# - Recursive configure doesn't work as $(top_srcdir)/git.mk inside the +# submodule doesn't find us. If you have configure.{in,ac} files in +# subdirs, add a proxy git.mk file in those dirs that simply does: +# "include $(top_srcdir)/../git.mk". Add more ..'s to your taste. +# And add those files to git. See vte/gnome-pty-helper/git.mk for +# example. +# + + + +############################################################################### +# Variables user modules may want to add to toplevel MAINTAINERCLEANFILES: +############################################################################### + +# +# Most autotools-using modules should be fine including this variable in their +# toplevel MAINTAINERCLEANFILES: +GITIGNORE_MAINTAINERCLEANFILES_TOPLEVEL = \ + $(srcdir)/aclocal.m4 \ + $(srcdir)/autoscan.log \ + $(srcdir)/configure.scan \ + `AUX_DIR=$(srcdir)/$$(cd $(top_srcdir); $(AUTOCONF) --trace 'AC_CONFIG_AUX_DIR:$$1' ./configure.ac); \ + test "x$$AUX_DIR" = "x$(srcdir)/" && AUX_DIR=$(srcdir); \ + for x in \ + ar-lib \ + compile \ + config.guess \ + config.sub \ + depcomp \ + install-sh \ + ltmain.sh \ + missing \ + mkinstalldirs \ + test-driver \ + ylwrap \ + ; do echo "$$AUX_DIR/$$x"; done` \ + `cd $(top_srcdir); $(AUTOCONF) --trace 'AC_CONFIG_HEADERS:$$1' ./configure.ac | \ + head -n 1 | while read f; do echo "$(srcdir)/$$f.in"; done` +# +# All modules should also be fine including the following variable, which +# removes automake-generated Makefile.in files: +GITIGNORE_MAINTAINERCLEANFILES_MAKEFILE_IN = \ + `cd $(top_srcdir); $(AUTOCONF) --trace 'AC_CONFIG_FILES:$$1' ./configure.ac | \ + while read f; do \ + case $$f in Makefile|*/Makefile) \ + test -f "$(srcdir)/$$f.am" && echo "$(srcdir)/$$f.in";; esac; \ + done` +# +# Modules that use libtool and use AC_CONFIG_MACRO_DIR() may also include this, +# though it's harmless to include regardless. +GITIGNORE_MAINTAINERCLEANFILES_M4_LIBTOOL = \ + `MACRO_DIR=$(srcdir)/$$(cd $(top_srcdir); $(AUTOCONF) --trace 'AC_CONFIG_MACRO_DIR:$$1' ./configure.ac); \ + if test "x$$MACRO_DIR" != "x$(srcdir)/"; then \ + for x in \ + libtool.m4 \ + ltoptions.m4 \ + ltsugar.m4 \ + ltversion.m4 \ + lt~obsolete.m4 \ + ; do echo "$$MACRO_DIR/$$x"; done; \ + fi` + + + +############################################################################### +# Default rule is to install ourselves in all Makefile.am files: +############################################################################### + +git-all: git-mk-install + +git-mk-install: + @echo "Installing git makefile" + @any_failed=; \ + find "`test -z "$(top_srcdir)" && echo . || echo "$(top_srcdir)"`" -name Makefile.am | while read x; do \ + if grep 'include .*/git.mk' $$x >/dev/null; then \ + echo "$$x already includes git.mk"; \ + else \ + failed=; \ + echo "Updating $$x"; \ + { cat $$x; \ + echo ''; \ + echo '-include $$(top_srcdir)/git.mk'; \ + } > $$x.tmp || failed=1; \ + if test x$$failed = x; then \ + mv $$x.tmp $$x || failed=1; \ + fi; \ + if test x$$failed = x; then : else \ + echo "Failed updating $$x"; >&2 \ + any_failed=1; \ + fi; \ + fi; done; test -z "$$any_failed" + +git-mk-update: + wget $(GIT_MK_URL) -O $(top_srcdir)/git.mk + +.PHONY: git-all git-mk-install git-mk-update + + + +############################################################################### +# Actual .gitignore generation: +############################################################################### + +$(srcdir)/.gitignore: Makefile.am $(top_srcdir)/git.mk + @echo "git.mk: Generating $@" + @{ \ + if test "x$(DOC_MODULE)" = x -o "x$(DOC_MAIN_SGML_FILE)" = x; then :; else \ + for x in \ + $(DOC_MODULE)-decl-list.txt \ + $(DOC_MODULE)-decl.txt \ + tmpl/$(DOC_MODULE)-unused.sgml \ + "tmpl/*.bak" \ + $(REPORT_FILES) \ + $(DOC_MODULE).pdf \ + xml html \ + ; do echo "/$$x"; done; \ + FLAVOR=$$(cd $(top_srcdir); $(AUTOCONF) --trace 'GTK_DOC_CHECK:$$2' ./configure.ac); \ + case $$FLAVOR in *no-tmpl*) echo /tmpl;; esac; \ + if echo "$(SCAN_OPTIONS)" | grep -q "\-\-rebuild-types"; then \ + echo "/$(DOC_MODULE).types"; \ + fi; \ + if echo "$(SCAN_OPTIONS)" | grep -q "\-\-rebuild-sections"; then \ + echo "/$(DOC_MODULE)-sections.txt"; \ + fi; \ + if test "$(abs_srcdir)" != "$(abs_builddir)" ; then \ + for x in \ + $(SETUP_FILES) \ + $(DOC_MODULE).types \ + ; do echo "/$$x"; done; \ + fi; \ + fi; \ + if test "x$(DOC_MODULE)$(DOC_ID)" = x -o "x$(DOC_LINGUAS)" = x; then :; else \ + for lc in $(DOC_LINGUAS); do \ + for x in \ + $(if $(DOC_MODULE),$(DOC_MODULE).xml) \ + $(DOC_PAGES) \ + $(DOC_INCLUDES) \ + ; do echo "/$$lc/$$x"; done; \ + done; \ + for x in \ + $(_DOC_OMF_ALL) \ + $(_DOC_DSK_ALL) \ + $(_DOC_HTML_ALL) \ + $(_DOC_MOFILES) \ + $(DOC_H_FILE) \ + "*/.xml2po.mo" \ + "*/*.omf.out" \ + ; do echo /$$x; done; \ + fi; \ + if test "x$(HELP_ID)" = x -o "x$(HELP_LINGUAS)" = x; then :; else \ + for lc in $(HELP_LINGUAS); do \ + for x in \ + $(HELP_FILES) \ + "$$lc.stamp" \ + "$$lc.mo" \ + ; do echo "/$$lc/$$x"; done; \ + done; \ + fi; \ + if test "x$(gsettings_SCHEMAS)" = x; then :; else \ + for x in \ + $(gsettings_SCHEMAS:.xml=.valid) \ + $(gsettings__enum_file) \ + ; do echo "/$$x"; done; \ + fi; \ + if test "x$(appdata_XML)" = x; then :; else \ + for x in \ + $(appdata_XML:.xml=.valid) \ + ; do echo "/$$x"; done; \ + fi; \ + if test "x$(appstream_XML)" = x; then :; else \ + for x in \ + $(appstream_XML:.xml=.valid) \ + ; do echo "/$$x"; done; \ + fi; \ + if test -f $(srcdir)/po/Makefile.in.in; then \ + for x in \ + po/Makefile.in.in \ + po/Makefile.in.in~ \ + po/Makefile.in \ + po/Makefile \ + po/Makevars.template \ + po/POTFILES \ + po/Rules-quot \ + po/stamp-it \ + po/stamp-po \ + po/.intltool-merge-cache \ + "po/*.gmo" \ + "po/*.header" \ + "po/*.mo" \ + "po/*.sed" \ + "po/*.sin" \ + po/$(GETTEXT_PACKAGE).pot \ + intltool-extract.in \ + intltool-merge.in \ + intltool-update.in \ + ; do echo "/$$x"; done; \ + fi; \ + if test -f $(srcdir)/configure; then \ + for x in \ + autom4te.cache \ + configure \ + config.h \ + stamp-h1 \ + libtool \ + config.lt \ + ; do echo "/$$x"; done; \ + fi; \ + if test "x$(DEJATOOL)" = x; then :; else \ + for x in \ + $(DEJATOOL) \ + ; do echo "/$$x.sum"; echo "/$$x.log"; done; \ + echo /site.exp; \ + fi; \ + if test "x$(am__dirstamp)" = x; then :; else \ + echo "$(am__dirstamp)"; \ + fi; \ + if test "x$(findstring libtool,$(LTCOMPILE))" = x -a "x$(findstring libtool,$(LTCXXCOMPILE))" = x -a "x$(GTKDOC_RUN)" = x; then :; else \ + for x in \ + "*.lo" \ + ".libs" "_libs" \ + ; do echo "$$x"; done; \ + fi; \ + for x in \ + .gitignore \ + $(GITIGNOREFILES) \ + $(CLEANFILES) \ + $(PROGRAMS) $(check_PROGRAMS) $(EXTRA_PROGRAMS) \ + $(LIBRARIES) $(check_LIBRARIES) $(EXTRA_LIBRARIES) \ + $(LTLIBRARIES) $(check_LTLIBRARIES) $(EXTRA_LTLIBRARIES) \ + so_locations \ + $(MOSTLYCLEANFILES) \ + $(TEST_LOGS) \ + $(TEST_LOGS:.log=.trs) \ + $(TEST_SUITE_LOG) \ + $(TESTS:=.test) \ + "*.gcda" \ + "*.gcno" \ + $(DISTCLEANFILES) \ + $(am__CONFIG_DISTCLEAN_FILES) \ + $(CONFIG_CLEAN_FILES) \ + TAGS ID GTAGS GRTAGS GSYMS GPATH tags \ + "*.tab.c" \ + $(MAINTAINERCLEANFILES) \ + $(BUILT_SOURCES) \ + $(patsubst %.vala,%.c,$(filter %.vala,$(SOURCES))) \ + $(filter %_vala.stamp,$(DIST_COMMON)) \ + $(filter %.vapi,$(DIST_COMMON)) \ + $(filter $(addprefix %,$(notdir $(patsubst %.vapi,%.h,$(filter %.vapi,$(DIST_COMMON))))),$(DIST_COMMON)) \ + Makefile \ + Makefile.in \ + "*.orig" \ + "*.rej" \ + "*.bak" \ + "*~" \ + ".*.sw[nop]" \ + ".dirstamp" \ + ; do echo "/$$x"; done; \ + for x in \ + "*.$(OBJEXT)" \ + $(DEPDIR) \ + ; do echo "$$x"; done; \ + } | \ + sed "s@^/`echo "$(srcdir)" | sed 's/\(.\)/[\1]/g'`/@/@" | \ + sed 's@/[.]/@/@g' | \ + LC_ALL=C sort | uniq > $@.tmp && \ + mv $@.tmp $@; + +all: $(srcdir)/.gitignore gitignore-recurse-maybe +gitignore: $(srcdir)/.gitignore gitignore-recurse + +gitignore-recurse-maybe: + @for subdir in $(DIST_SUBDIRS); do \ + case " $(SUBDIRS) " in \ + *" $$subdir "*) :;; \ + *) test "$$subdir" = . -o -e "$$subdir/.git" || (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) gitignore || echo "Skipping $$subdir");; \ + esac; \ + done +gitignore-recurse: + @for subdir in $(DIST_SUBDIRS); do \ + test "$$subdir" = . -o -e "$$subdir/.git" || (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) gitignore || echo "Skipping $$subdir"); \ + done + +maintainer-clean: gitignore-clean +gitignore-clean: + -rm -f $(srcdir)/.gitignore + +.PHONY: gitignore-clean gitignore gitignore-recurse gitignore-recurse-maybe From 91734a8a18e714d68529ea0fecbc526dafe9977e Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 7 Apr 2016 17:03:08 +0200 Subject: [PATCH 47/69] Support pathnames for --subpath=... This allows you to pull a single file, rather than just a directory. Closes: #244 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 15 +++++++++------ tests/test-pull-subpath.sh | 4 ++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 4d8b6138..a6272b17 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -448,12 +448,7 @@ scan_dirtree_object (OtPullData *pull_data, files_variant = g_variant_get_child_value (tree, 0); dirs_variant = g_variant_get_child_value (tree, 1); - /* Skip files if we're traversing a request only directory */ - if (pull_data->dir) - n = 0; - else - n = g_variant_n_children (files_variant); - + n = g_variant_n_children (files_variant); for (i = 0; i < n; i++) { const char *filename; @@ -466,6 +461,14 @@ scan_dirtree_object (OtPullData *pull_data, if (!ot_util_filename_validate (filename, error)) goto out; + /* Skip files if we're traversing a request only directory, unless it exactly + * matches the path */ + if (pull_data->dir && + /* Should always an initial slash, we assert it in scan_dirtree_object */ + pull_data->dir[0] == '/' && + strcmp (pull_data->dir+1, filename) != 0) + continue; + file_checksum = ostree_checksum_from_bytes_v (csum); if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, file_checksum, diff --git a/tests/test-pull-subpath.sh b/tests/test-pull-subpath.sh index 8d20341e..b0bf4839 100755 --- a/tests/test-pull-subpath.sh +++ b/tests/test-pull-subpath.sh @@ -44,6 +44,10 @@ assert_file_has_content err.txt "Couldn't find file object" rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) assert_has_file repo/state/${rev}.commitpartial +# Test pulling a file, not a dir +${CMD_PREFIX} ostree --repo=repo pull --subpath=/firstfile origin main +${CMD_PREFIX} ostree --repo=repo ls origin:main /firstfile + ${CMD_PREFIX} ostree --repo=repo pull origin main assert_not_has_file repo/state/${rev}.commitpartial ${CMD_PREFIX} ostree --repo=repo fsck From 37382590dc66791f0cdaff876fba24875e9155af Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 31 Mar 2016 22:00:43 +0200 Subject: [PATCH 48/69] Export ostree_repo_get_remote_option* functions These are useful for ostree users (like xdg-app) that have custom options for remotes. In particular they are useful when we later make them all respect self->parent_repo. Closes: #236 Approved by: cgwalters --- src/libostree/libostree.sym | 3 + src/libostree/ostree-repo-private.h | 23 ----- src/libostree/ostree-repo-pull.c | 12 +-- src/libostree/ostree-repo.c | 131 +++++++++++++++++++--------- src/libostree/ostree-repo.h | 23 +++++ 5 files changed, 122 insertions(+), 70 deletions(-) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index ca3c6fd0..c4f44d39 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -321,6 +321,9 @@ global: ostree_sysroot_deployment_unlock; ostree_deployment_get_unlocked; ostree_deployment_unlocked_state_to_string; + ostree_repo_get_remote_option; + ostree_repo_get_remote_list_option; + ostree_repo_get_remote_boolean_option; } LIBOSTREE_2016.3; /* NOTE NOTE NOTE diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 484a6ec8..24380b21 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -195,29 +195,6 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, gboolean _ostree_repo_remote_name_is_file (const char *remote_name); -gboolean -_ostree_repo_get_remote_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - const char *default_value, - char **out_value, - GError **error); - -gboolean -_ostree_repo_get_remote_list_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - char ***out_value, - GError **error); - -gboolean -_ostree_repo_get_remote_boolean_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - gboolean default_value, - gboolean *out_value, - GError **error); - gboolean _ostree_repo_get_remote_option_inherit (OstreeRepo *self, const char *remote_name, diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index a6272b17..8eca91c3 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -2008,9 +2008,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - if (!_ostree_repo_get_remote_option (self, - remote_name_or_baseurl, "metalink", - NULL, &metalink_url_str, error)) + if (!ostree_repo_get_remote_option (self, + remote_name_or_baseurl, "metalink", + NULL, &metalink_url_str, error)) goto out; if (!metalink_url_str) @@ -2064,9 +2064,9 @@ ostree_repo_pull_with_options (OstreeRepo *self, summary_bytes, FALSE); } - if (!_ostree_repo_get_remote_list_option (self, - remote_name_or_baseurl, "branches", - &configured_branches, error)) + if (!ostree_repo_get_remote_list_option (self, + remote_name_or_baseurl, "branches", + &configured_branches, error)) goto out; if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index a5d0bb2d..81b68a08 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -257,13 +257,29 @@ _ostree_repo_remote_name_is_file (const char *remote_name) return g_str_has_prefix (remote_name, "file://"); } +/** + * ostree_repo_get_remote_option: + * @self: A OstreeRepo + * @remote_name: Name + * @option_name: Option + * @default_value: (allow-none): Value returned if @option_name is not present + * @out_value: (out): Return location for value + * @error: Error + * + * OSTree remotes are represented by keyfile groups, formatted like: + * `[remote "remotename"]`. This function returns a value named @option_name + * underneath that group, or @default_value if the remote exists but not the + * option name. + * + * Returns: %TRUE on success, otherwise %FALSE with @error set + */ gboolean -_ostree_repo_get_remote_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - const char *default_value, - char **out_value, - GError **error) +ostree_repo_get_remote_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + const char *default_value, + char **out_value, + GError **error) { local_cleanup_remote OstreeRemote *remote = NULL; gboolean ret = FALSE; @@ -289,12 +305,29 @@ _ostree_repo_get_remote_option (OstreeRepo *self, return ret; } +/** + * ostree_repo_get_remote_list_option: + * @self: A OstreeRepo + * @remote_name: Name + * @option_name: Option + * @out_value: (out) (array zero-terminated=1): location to store the list + * of strings. The list should be freed with + * g_strfreev(). + * @error: Error + * + * OSTree remotes are represented by keyfile groups, formatted like: + * `[remote "remotename"]`. This function returns a value named @option_name + * underneath that group, and returns it as an zero terminated array of strings. + * If the option is not set, @out_value will be set to %NULL. + * + * Returns: %TRUE on success, otherwise %FALSE with @error set + */ gboolean -_ostree_repo_get_remote_list_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - char ***out_value, - GError **error) +ostree_repo_get_remote_list_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + char ***out_value, + GError **error) { local_cleanup_remote OstreeRemote *remote = NULL; gboolean ret = FALSE; @@ -331,13 +364,29 @@ _ostree_repo_get_remote_list_option (OstreeRepo *self, return ret; } +/** + * ostree_repo_get_remote_boolean_option: + * @self: A OstreeRepo + * @remote_name: Name + * @option_name: Option + * @default_value: (allow-none): Value returned if @option_name is not present + * @out_value: (out) : location to store the result. + * @error: Error + * + * OSTree remotes are represented by keyfile groups, formatted like: + * `[remote "remotename"]`. This function returns a value named @option_name + * underneath that group, and returns it as a boolean. + * If the option is not set, @out_value will be set to @default_value. + * + * Returns: %TRUE on success, otherwise %FALSE with @error set + */ gboolean -_ostree_repo_get_remote_boolean_option (OstreeRepo *self, - const char *remote_name, - const char *option_name, - gboolean default_value, - gboolean *out_value, - GError **error) +ostree_repo_get_remote_boolean_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + gboolean default_value, + gboolean *out_value, + GError **error) { local_cleanup_remote OstreeRemote *remote = NULL; gboolean ret = FALSE; @@ -374,9 +423,9 @@ _ostree_repo_get_remote_option_inherit (OstreeRepo *self, g_autofree char *value = NULL; gboolean ret = FALSE; - if (!_ostree_repo_get_remote_option (self, - remote_name, option_name, - NULL, &value, error)) + if (!ostree_repo_get_remote_option (self, + remote_name, option_name, + NULL, &value, error)) goto out; if (value == NULL && parent != NULL) @@ -412,9 +461,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, g_return_val_if_fail (OSTREE_IS_REPO (self), NULL); g_return_val_if_fail (remote_name != NULL, NULL); - if (!_ostree_repo_get_remote_boolean_option (self, remote_name, - "tls-permissive", FALSE, - &tls_permissive, error)) + if (!ostree_repo_get_remote_boolean_option (self, remote_name, + "tls-permissive", FALSE, + &tls_permissive, error)) goto out; if (tls_permissive) @@ -426,13 +475,13 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, g_autofree char *tls_client_cert_path = NULL; g_autofree char *tls_client_key_path = NULL; - if (!_ostree_repo_get_remote_option (self, remote_name, - "tls-client-cert-path", NULL, - &tls_client_cert_path, error)) + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-client-cert-path", NULL, + &tls_client_cert_path, error)) goto out; - if (!_ostree_repo_get_remote_option (self, remote_name, - "tls-client-key-path", NULL, - &tls_client_key_path, error)) + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-client-key-path", NULL, + &tls_client_key_path, error)) goto out; if ((tls_client_cert_path != NULL) != (tls_client_key_path != NULL)) @@ -462,9 +511,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, { g_autofree char *tls_ca_path = NULL; - if (!_ostree_repo_get_remote_option (self, remote_name, - "tls-ca-path", NULL, - &tls_ca_path, error)) + if (!ostree_repo_get_remote_option (self, remote_name, + "tls-ca-path", NULL, + &tls_ca_path, error)) goto out; if (tls_ca_path != NULL) @@ -482,9 +531,9 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self, { g_autofree char *http_proxy = NULL; - if (!_ostree_repo_get_remote_option (self, remote_name, - "proxy", NULL, - &http_proxy, error)) + if (!ostree_repo_get_remote_option (self, remote_name, + "proxy", NULL, + &http_proxy, error)) goto out; if (http_proxy != NULL) @@ -1343,8 +1392,8 @@ ostree_repo_remote_get_gpg_verify (OstreeRepo *self, return TRUE; } - return _ostree_repo_get_remote_boolean_option (self, name, "gpg-verify", - TRUE, out_gpg_verify, error); + return ostree_repo_get_remote_boolean_option (self, name, "gpg-verify", + TRUE, out_gpg_verify, error); } /** @@ -1366,8 +1415,8 @@ ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, gboolean *out_gpg_verify_summary, GError **error) { - return _ostree_repo_get_remote_boolean_option (self, name, "gpg-verify-summary", - FALSE, out_gpg_verify_summary, error); + return ostree_repo_get_remote_boolean_option (self, name, "gpg-verify-summary", + FALSE, out_gpg_verify_summary, error); } /** @@ -1889,8 +1938,8 @@ ostree_repo_remote_fetch_summary (OstreeRepo *self, g_return_val_if_fail (OSTREE_REPO (self), FALSE); g_return_val_if_fail (name != NULL, FALSE); - if (!_ostree_repo_get_remote_option (self, name, "metalink", NULL, - &metalink_url_string, error)) + if (!ostree_repo_get_remote_option (self, name, "metalink", NULL, + &metalink_url_string, error)) goto out; if (!repo_remote_fetch_summary (self, diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index f83aef23..822014f2 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -147,6 +147,29 @@ gboolean ostree_repo_remote_get_gpg_verify_summary (OstreeRepo *self, gboolean *out_gpg_verify_summary, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_get_remote_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + const char *default_value, + char **out_value, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_repo_get_remote_list_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + char ***out_value, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_repo_get_remote_boolean_option (OstreeRepo *self, + const char *remote_name, + const char *option_name, + gboolean default_value, + gboolean *out_value, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_remote_gpg_import (OstreeRepo *self, const char *name, From d7181f22a61df6f7661d666745a06a28f9d9303f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 31 Mar 2016 23:03:53 +0200 Subject: [PATCH 49/69] Inherit remotes and remote options from parent repo Closes: #236 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 7 - src/libostree/ostree-repo.c | 232 +++++++++++++++++++--------- 2 files changed, 159 insertions(+), 80 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 24380b21..c60ab246 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -195,13 +195,6 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self, gboolean _ostree_repo_remote_name_is_file (const char *remote_name); -gboolean -_ostree_repo_get_remote_option_inherit (OstreeRepo *self, - const char *remote_name, - const char *option_name, - char **out_value, - GError **error); - #ifdef HAVE_LIBSOUP OstreeFetcher * _ostree_repo_remote_new_fetcher (OstreeRepo *self, diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 81b68a08..47519e66 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -217,6 +217,27 @@ ost_repo_get_remote (OstreeRepo *self, return remote; } +static OstreeRemote * +ost_repo_get_remote_inherited (OstreeRepo *self, + const char *name, + GError **error) +{ + local_cleanup_remote OstreeRemote *remote = NULL; + g_autoptr(GError) temp_error = NULL; + + remote = ost_repo_get_remote (self, name, &temp_error); + if (remote == NULL) + { + if (self->parent_repo != NULL) + return ost_repo_get_remote_inherited (self->parent_repo, name, error); + + g_propagate_error (error, g_steal_pointer (&temp_error)); + return NULL; + } + + return g_steal_pointer (&remote); +} + static void ost_repo_add_remote (OstreeRepo *self, OstreeRemote *remote) @@ -283,6 +304,8 @@ ostree_repo_get_remote_option (OstreeRepo *self, { local_cleanup_remote OstreeRemote *remote = NULL; gboolean ret = FALSE; + g_autoptr(GError) temp_error = NULL; + g_autofree char *value = NULL; if (_ostree_repo_remote_name_is_file (remote_name)) { @@ -290,18 +313,40 @@ ostree_repo_get_remote_option (OstreeRepo *self, return TRUE; } - remote = ost_repo_get_remote (self, remote_name, error); - + remote = ost_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { - ret = ot_keyfile_get_value_with_default (remote->options, - remote->group, - option_name, - default_value, - out_value, - error); - } + value = g_key_file_get_string (remote->options, remote->group, option_name, &temp_error); + if (value == NULL) + { + if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + { + if (self->parent_repo != NULL) + return ostree_repo_get_remote_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + error); + value = g_strdup (default_value); + ret = TRUE; + } + else + g_propagate_error (error, g_steal_pointer (&temp_error)); + } + else + ret = TRUE; + } + else if (self->parent_repo != NULL) + return ostree_repo_get_remote_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + error); + else + g_propagate_error (error, g_steal_pointer (&temp_error)); + + *out_value = g_steal_pointer (&value); return ret; } @@ -331,6 +376,8 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, { local_cleanup_remote OstreeRemote *remote = NULL; gboolean ret = FALSE; + g_autoptr(GError) temp_error = NULL; + g_auto(GStrv) value = NULL; if (_ostree_repo_remote_name_is_file (remote_name)) { @@ -338,29 +385,38 @@ ostree_repo_get_remote_list_option (OstreeRepo *self, return TRUE; } - remote = ost_repo_get_remote (self, remote_name, error); - + remote = ost_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { - g_auto(GStrv) value = NULL; - GError *local_error = NULL; - value = g_key_file_get_string_list (remote->options, remote->group, option_name, - NULL, &local_error); + NULL, &temp_error); /* Default value if key not found is always NULL. */ - if (g_error_matches (local_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) - g_clear_error (&local_error); - - if (local_error == NULL) + if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) { - ot_transfer_out_value (out_value, &value); + if (self->parent_repo != NULL) + return ostree_repo_get_remote_list_option (self->parent_repo, + remote_name, option_name, + out_value, + error); ret = TRUE; } + else if (temp_error) + g_propagate_error (error, g_steal_pointer (&temp_error)); + else + ret = TRUE; } + else if (self->parent_repo != NULL) + return ostree_repo_get_remote_list_option (self->parent_repo, + remote_name, option_name, + out_value, + error); + else + g_propagate_error (error, g_steal_pointer (&temp_error)); + *out_value = g_steal_pointer (&value); return ret; } @@ -389,61 +445,50 @@ ostree_repo_get_remote_boolean_option (OstreeRepo *self, GError **error) { local_cleanup_remote OstreeRemote *remote = NULL; + g_autoptr(GError) temp_error = NULL; gboolean ret = FALSE; + gboolean value = FALSE; if (_ostree_repo_remote_name_is_file (remote_name)) { *out_value = default_value; return TRUE; } - - remote = ost_repo_get_remote (self, remote_name, error); + remote = ost_repo_get_remote (self, remote_name, &temp_error); if (remote != NULL) { - ret = ot_keyfile_get_boolean_with_default (remote->options, - remote->group, - option_name, - default_value, - out_value, - error); + value = g_key_file_get_boolean (remote->options, remote->group, option_name, &temp_error); + if (temp_error != NULL) + { + if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND)) + { + if (self->parent_repo != NULL) + return ostree_repo_get_remote_boolean_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + error); + + value = default_value; + ret = TRUE; + } + else + g_propagate_error (error, g_steal_pointer (&temp_error)); + } + else + ret = TRUE; } + else if (self->parent_repo != NULL) + return ostree_repo_get_remote_boolean_option (self->parent_repo, + remote_name, option_name, + default_value, + out_value, + error); + else + g_propagate_error (error, g_steal_pointer (&temp_error)); - return ret; -} - -gboolean -_ostree_repo_get_remote_option_inherit (OstreeRepo *self, - const char *remote_name, - const char *option_name, - char **out_value, - GError **error) -{ - OstreeRepo *parent = ostree_repo_get_parent (self); - g_autofree char *value = NULL; - gboolean ret = FALSE; - - if (!ostree_repo_get_remote_option (self, - remote_name, option_name, - NULL, &value, error)) - goto out; - - if (value == NULL && parent != NULL) - { - if (!_ostree_repo_get_remote_option_inherit (parent, - remote_name, option_name, - &value, error)) - goto out; - } - - /* Success here just means no error occurred during lookup, - * not necessarily that we found a value for the option name. */ - if (out_value != NULL) - *out_value = g_steal_pointer (&value); - - ret = TRUE; - -out: + *out_value = value; return ret; } @@ -1265,6 +1310,25 @@ ostree_repo_remote_change (OstreeRepo *self, g_assert_not_reached (); } +static void +_ostree_repo_remote_list (OstreeRepo *self, + GHashTable *out) +{ + GHashTableIter iter; + gpointer key, value; + + g_mutex_lock (&self->remotes_lock); + + g_hash_table_iter_init (&iter, self->remotes); + while (g_hash_table_iter_next (&iter, &key, &value)) + g_hash_table_insert (out, g_strdup (key), NULL); + + g_mutex_unlock (&self->remotes_lock); + + if (self->parent_repo) + _ostree_repo_remote_list (self, out); +} + /** * ostree_repo_remote_list: * @self: Repo @@ -1282,10 +1346,15 @@ ostree_repo_remote_list (OstreeRepo *self, { char **remotes = NULL; guint n_remotes; + g_autoptr(GHashTable) remotes_ht = NULL; - g_mutex_lock (&self->remotes_lock); + remotes_ht = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify) g_free, + (GDestroyNotify) NULL); - n_remotes = g_hash_table_size (self->remotes); + _ostree_repo_remote_list (self, remotes_ht); + + n_remotes = g_hash_table_size (remotes_ht); if (n_remotes > 0) { @@ -1294,7 +1363,7 @@ ostree_repo_remote_list (OstreeRepo *self, remotes = g_new (char *, n_remotes + 1); - list = g_hash_table_get_keys (self->remotes); + list = g_hash_table_get_keys (remotes_ht); list = g_list_sort (list, (GCompareFunc) strcmp); for (link = list; link != NULL; link = link->next) @@ -1305,8 +1374,6 @@ ostree_repo_remote_list (OstreeRepo *self, remotes[ii] = NULL; } - g_mutex_unlock (&self->remotes_lock); - if (out_n_remotes) *out_n_remotes = n_remotes; @@ -1342,7 +1409,7 @@ ostree_repo_remote_get_url (OstreeRepo *self, } else { - if (!_ostree_repo_get_remote_option_inherit (self, name, "url", &url, error)) + if (!ostree_repo_get_remote_option (self, name, "url", NULL, &url, error)) goto out; if (url == NULL) @@ -1469,7 +1536,7 @@ ostree_repo_remote_gpg_import (OstreeRepo *self, /* First make sure the remote name is valid. */ - remote = ost_repo_get_remote (self, name, error); + remote = ost_repo_get_remote_inherited (self, name, error); if (remote == NULL) goto out; @@ -4410,6 +4477,25 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, /* Special remote for _ostree_repo_gpg_verify_with_metadata() */ static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__"; +static GFile * +find_keyring (OstreeRepo *self, + OstreeRemote *remote, + GCancellable *cancellable) +{ + g_autoptr(GFile) file = NULL; + file = g_file_get_child (self->repodir, remote->keyring); + + if (g_file_query_exists (file, cancellable)) + { + return g_steal_pointer (&file); + } + + if (self->parent_repo) + return find_keyring (self->parent_repo, remote, cancellable); + + return NULL; +} + OstreeGpgVerifyResult * _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, GBytes *signed_data, @@ -4446,13 +4532,13 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self, OstreeRemote *remote; g_autoptr(GFile) file = NULL; - remote = ost_repo_get_remote (self, remote_name, error); + remote = ost_repo_get_remote_inherited (self, remote_name, error); if (remote == NULL) goto out; - file = g_file_get_child (self->repodir, remote->keyring); + file = find_keyring (self, remote, cancellable); - if (g_file_query_exists (file, cancellable)) + if (file != NULL) { _ostree_gpg_verifier_add_keyring (verifier, file); add_global_keyring_dir = FALSE; From 6aa447a917c02d67dc65b41b3716c9951544b110 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 1 Apr 2016 10:20:34 +0200 Subject: [PATCH 50/69] Add test case for inheriting remote options Closes: #236 Approved by: cgwalters --- Makefile-tests.am | 1 + tests/test-parent.sh | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100755 tests/test-parent.sh diff --git a/Makefile-tests.am b/Makefile-tests.am index 4adfbf7b..b3d75142 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -40,6 +40,7 @@ test_scripts = \ tests/test-export.sh \ tests/test-help.sh \ tests/test-libarchive.sh \ + tests/test-parent.sh \ tests/test-pull-archive-z.sh \ tests/test-pull-commit-only.sh \ tests/test-pull-corruption.sh \ diff --git a/tests/test-parent.sh b/tests/test-parent.sh new file mode 100755 index 00000000..05b102a2 --- /dev/null +++ b/tests/test-parent.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2016 Alexander Larsson +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +echo '1..2' + +setup_test_repository "archive-z2" + +export OSTREE_GPG_SIGN="${OSTREE} gpg-sign --gpg-homedir=${TEST_GPG_KEYHOME}" + +cd ${test_tmpdir} + +# Create a repo +${CMD_PREFIX} ostree --repo=repo2 init +${CMD_PREFIX} ostree --repo=repo2 remote add --gpg-import=${test_tmpdir}/gpghome/trusted/pubring.gpg --set=gpg-verify=true aremote file://$(pwd)/repo test2 + +# Create a repo with repo2 as parent +${CMD_PREFIX} ostree init --repo=repo3 --mode=bare-user +${CMD_PREFIX} ostree config --repo=repo3 set core.parent `pwd`/repo2 + +# Ensure the unsigned pull fails so we know we imported the gpg config correctly +if ${CMD_PREFIX} ostree --repo=repo3 pull aremote; then + assert_not_reached "GPG verification unexpectedly succeeded" +fi +echo "ok unsigned pull w/parent" + +# Make a signed commit and ensure we can now pull +${OSTREE} commit -b test2 -s "A GPG signed commit" -m "Signed commit body" --gpg-sign=${TEST_GPG_KEYID_1} --gpg-homedir=${TEST_GPG_KEYHOME} --tree=dir=files +${CMD_PREFIX} ostree --repo=repo3 pull aremote + +echo "ok signed pull w/parent" From 8ff9a48ce39d1e3f0bdd01451fd9627a16af5c14 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Fri, 8 Apr 2016 12:10:44 +0200 Subject: [PATCH 51/69] cfg.mk: ignore syntax-check for git.mk Signed-off-by: Giuseppe Scrivano Closes: #246 Approved by: cgwalters --- cfg.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfg.mk b/cfg.mk index 5c3f0b40..3806209e 100644 --- a/cfg.mk +++ b/cfg.mk @@ -1,4 +1,4 @@ -export VC_LIST_EXCEPT_DEFAULT=^(lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|buildutil/.*)$$ +export VC_LIST_EXCEPT_DEFAULT=^(git.mk|lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|buildutil/.*)$$ local-checks-to-skip = \ sc_const_long_option \ From ddda8e5b8bb1ec21b2a4c5340dc36362c59c1271 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 8 Apr 2016 10:44:24 +0200 Subject: [PATCH 52/69] Add support for ostree static-delta delete Closes: #245 Approved by: giuseppe --- man/ostree-static-delta.xml | 6 ++++ src/libostree/ostree-cmdprivate.c | 3 +- src/libostree/ostree-cmdprivate.h | 1 + src/libostree/ostree-repo-static-delta-core.c | 35 +++++++++++++++++++ .../ostree-repo-static-delta-private.h | 6 ++++ src/ostree/ot-builtin-static-delta.c | 35 +++++++++++++++++++ tests/test-delta.sh | 17 ++++++++- 7 files changed, 101 insertions(+), 2 deletions(-) diff --git a/man/ostree-static-delta.xml b/man/ostree-static-delta.xml index d410a2d6..ed2e1f48 100644 --- a/man/ostree-static-delta.xml +++ b/man/ostree-static-delta.xml @@ -51,6 +51,12 @@ Boston, MA 02111-1307, USA. ostree static-delta list + + ostree static-delta show + + + ostree static-delta delete + ostree static-delta generate --to=REV OPTIONS diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index 74c32a3c..2c85bb4a 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -46,7 +46,8 @@ ostree_cmd__private__ (void) { static OstreeCmdPrivateVTable table = { impl_ostree_generate_grub2_config, - _ostree_repo_static_delta_dump + _ostree_repo_static_delta_dump, + _ostree_repo_static_delta_delete }; return &table; diff --git a/src/libostree/ostree-cmdprivate.h b/src/libostree/ostree-cmdprivate.h index 81061568..9a74ead1 100644 --- a/src/libostree/ostree-cmdprivate.h +++ b/src/libostree/ostree-cmdprivate.h @@ -27,6 +27,7 @@ G_BEGIN_DECLS typedef struct { gboolean (* ostree_generate_grub2_config) (OstreeSysroot *sysroot, int bootversion, int target_fd, GCancellable *cancellable, GError **error); gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); + gboolean (* ostree_static_delta_delete) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); } OstreeCmdPrivateVTable; /* Note this not really "public", we just export the symbol, but not the header */ diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index d84f0019..9e3ed09f 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -782,6 +782,41 @@ _ostree_delta_needs_byteswap (GVariant *superblock) } } +gboolean +_ostree_repo_static_delta_delete (OstreeRepo *self, + const char *delta_id, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *from = NULL; + g_autofree char *to = NULL; + g_autofree char *deltadir = NULL; + struct stat buf; + + _ostree_parse_delta_name (delta_id, &from, &to); + deltadir = _ostree_get_relative_static_delta_path (from, to, NULL); + + if (fstatat (self->repo_dir_fd, deltadir, &buf, 0) != 0) + { + if (errno == ENOENT) + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Can't find delta %s", delta_id); + else + glnx_set_error_from_errno (error); + + goto out; + } + + if (!glnx_shutil_rm_rf_at (self->repo_dir_fd, deltadir, + cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +} + gboolean _ostree_repo_static_delta_dump (OstreeRepo *self, const char *delta_id, diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 41ddad48..eeb99c3f 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -196,6 +196,12 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_static_delta_delete (OstreeRepo *repo, + const char *delta_id, + GCancellable *cancellable, + GError **error); + /* Used for static deltas which due to a historical mistake are * inconsistent endian. * diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 36fb63fa..09eb90ab 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -42,6 +42,7 @@ static gboolean opt_disable_bsdiff; BUILTINPROTO(list); BUILTINPROTO(show); +BUILTINPROTO(delete); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); @@ -50,6 +51,7 @@ BUILTINPROTO(apply_offline); static OstreeCommand static_delta_subcommands[] = { { "list", ot_static_delta_builtin_list }, { "show", ot_static_delta_builtin_show }, + { "delete", ot_static_delta_builtin_delete }, { "generate", ot_static_delta_builtin_generate }, { "apply-offline", ot_static_delta_builtin_apply_offline }, { NULL, NULL } @@ -168,6 +170,39 @@ ot_static_delta_builtin_show (int argc, char **argv, GCancellable *cancellable, return ret; } +static gboolean +ot_static_delta_builtin_delete (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + const char *delta_id = NULL; + + context = g_option_context_new ("DELETE - Remove a delta"); + + if (!ostree_option_context_parse (context, list_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + + if (argc < 3) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "DELTA must be specified"); + goto out; + } + + delta_id = argv[2]; + + if (!ostree_cmd__private__ ()->ostree_static_delta_delete (repo, delta_id, cancellable, error)) + goto out; + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} + + static gboolean ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellable, GError **error) { diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 411e9b04..4b2b879a 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -26,7 +26,7 @@ skip_without_user_xattrs bindatafiles="bash true ostree" morebindatafiles="false ls" -echo '1..7' +echo '1..8' mkdir repo ${CMD_PREFIX} ostree --repo=repo init --mode=archive-z2 @@ -186,3 +186,18 @@ ${CMD_PREFIX} ostree --repo=repo2 fsck ${CMD_PREFIX} ostree --repo=repo2 ls ${newrev} >/dev/null echo 'ok apply offline inline' + +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}-${newrev}$ || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}$ || exit 1 + +${CMD_PREFIX} ostree --repo=repo static-delta delete ${origrev} || exit 1 + +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}-${newrev}$ || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}$ && exit 1 + +${CMD_PREFIX} ostree --repo=repo static-delta delete ${origrev}-${newrev} || exit 1 + +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}-${newrev}$ && exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta list | grep ^${origrev}$ && exit 1 + +echo 'ok delete' From 41661e47e18856a9886fa37bdb83c8765ed057be Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 8 Apr 2016 12:18:17 -0400 Subject: [PATCH 53/69] small cleanups - Revert 'cannot' --> 'can not' (it's the exception!) - Remove duplicate function - Squelch compiler warnings Closes: #248 Approved by: cgwalters --- cfg.mk | 2 +- docs/manual/related-projects.md | 2 +- src/libostree/ostree-metalink.c | 4 +-- src/libostree/ostree-repo-file.c | 50 ++------------------------------ src/libotutil/ot-variant-utils.c | 4 +-- 5 files changed, 8 insertions(+), 54 deletions(-) diff --git a/cfg.mk b/cfg.mk index 3806209e..d6a5039d 100644 --- a/cfg.mk +++ b/cfg.mk @@ -1,4 +1,4 @@ -export VC_LIST_EXCEPT_DEFAULT=^(git.mk|lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|buildutil/.*)$$ +export VC_LIST_EXCEPT_DEFAULT=^(docs/.*|git.mk|lib/.*|m4/.*|md5/.*|build-aux/.*|src/gettext\.h|.*ChangeLog|buildutil/.*)$$ local-checks-to-skip = \ sc_const_long_option \ diff --git a/docs/manual/related-projects.md b/docs/manual/related-projects.md index 73972330..896c7655 100644 --- a/docs/manual/related-projects.md +++ b/docs/manual/related-projects.md @@ -63,7 +63,7 @@ awareness of BTRFS in dpkg/rpm itself) will be required. The OSTree author believes that having total freedom at the block storage layer is better for general purpose operating systems. For example, with OSTree, one is free to use BTRFS in any way you like - -you can use a subvolume for `/home`, or you cannot. +you may decide to use a subvolume for `/home`, or not. Furthermore, in its most basic incarnation, the rpm/dpkg + BTRFS doesn't solve the race conditions that happen when unpacking packages diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c index 5ca69e69..ad3a6a2a 100644 --- a/src/libostree/ostree-metalink.c +++ b/src/libostree/ostree-metalink.c @@ -491,7 +491,7 @@ try_metalink_targets (OstreeMetalinkRequest *self, GError **error) { gboolean ret = FALSE; - SoupURI *target_uri; + SoupURI *target_uri = NULL; if (!self->found_a_file_element) { @@ -564,7 +564,7 @@ try_metalink_targets (OstreeMetalinkRequest *self, self->urls->len, self->last_metalink_error); goto out; } - + ret = TRUE; if (out_target_uri) *out_target_uri = soup_uri_copy (target_uri); diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c index 49f7333d..396820da 100644 --- a/src/libostree/ostree-repo-file.c +++ b/src/libostree/ostree-repo-file.c @@ -745,52 +745,6 @@ query_child_info_dir (OstreeRepo *repo, return ret; } -static gboolean -bsearch_in_file_variant (GVariant *variant, - const char *name, - int *out_pos) -{ - gsize imax, imin; - gsize imid; - gsize n; - - n = g_variant_n_children (variant); - if (n == 0) - return FALSE; - - imax = n - 1; - imin = 0; - while (imax >= imin) - { - g_autoptr(GVariant) child = NULL; - const char *cur; - int cmp; - - imid = (imin + imax) / 2; - - child = g_variant_get_child_value (variant, imid); - g_variant_get_child (child, 0, "&s", &cur, NULL); - - cmp = strcmp (cur, name); - if (cmp < 0) - imin = imid + 1; - else if (cmp > 0) - { - if (imid == 0) - break; - imax = imid - 1; - } - else - { - *out_pos = imid; - return TRUE; - } - } - - *out_pos = imid; - return FALSE; -} - int ostree_repo_file_tree_find_child (OstreeRepoFile *self, const char *name, @@ -806,7 +760,7 @@ ostree_repo_file_tree_find_child (OstreeRepoFile *self, dirs_variant = g_variant_get_child_value (self->tree_contents, 1); i = -1; - if (bsearch_in_file_variant (files_variant, name, &i)) + if (ot_variant_bsearch_str (files_variant, name, &i)) { *is_dir = FALSE; ret_container = files_variant; @@ -814,7 +768,7 @@ ostree_repo_file_tree_find_child (OstreeRepoFile *self, } else { - if (bsearch_in_file_variant (dirs_variant, name, &i)) + if (ot_variant_bsearch_str (dirs_variant, name, &i)) { *is_dir = TRUE; ret_container = dirs_variant; diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 0305c34d..2dc07582 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -319,7 +319,7 @@ ot_variant_bsearch_str (GVariant *array, int *out_pos) { gsize imax, imin; - gsize imid; + gsize imid = -1; gsize n; n = g_variant_n_children (array); @@ -337,7 +337,7 @@ ot_variant_bsearch_str (GVariant *array, imid = (imin + imax) / 2; child = g_variant_get_child_value (array, imid); - g_variant_get_child (child, 0, "&s", &cur, NULL); + g_variant_get_child (child, 0, "&s", &cur, NULL); cmp = strcmp (cur, str); if (cmp < 0) From e62beae6953c344761af680b6acb21ab8a967a90 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 11 Apr 2016 12:46:57 +0200 Subject: [PATCH 54/69] Fix the symbol versions for ostree_repo_get_remote_*option These were accidentally added to 2016.4 instead of 2016.5 Closes: #251 Approved by: cgwalters --- src/libostree/libostree.sym | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index c4f44d39..ea18632f 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -321,9 +321,6 @@ global: ostree_sysroot_deployment_unlock; ostree_deployment_get_unlocked; ostree_deployment_unlocked_state_to_string; - ostree_repo_get_remote_option; - ostree_repo_get_remote_list_option; - ostree_repo_get_remote_boolean_option; } LIBOSTREE_2016.3; /* NOTE NOTE NOTE @@ -335,4 +332,7 @@ LIBOSTREE_2016.5 { global: ostree_repo_import_object_from_with_trust; ostree_sepolicy_get_csum; + ostree_repo_get_remote_option; + ostree_repo_get_remote_list_option; + ostree_repo_get_remote_boolean_option; } LIBOSTREE_2016.4; From b69fd11487cfa23ff930da6b80b73e03b0daabe3 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Mon, 11 Apr 2016 09:27:35 -0400 Subject: [PATCH 55/69] ostree-repo-pull: always initialize flags_i Otherwise we get undefined behaviour if the client didn't explicitly set any flags. Also, add documentation for all the other options supported by ostree_repo_pull_with_options(). Closes: #252 Approved by: cgwalters --- src/libostree/ostree-repo-pull.c | 2 +- src/libostree/ostree-repo.c | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 8eca91c3..e86ede50 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1908,7 +1908,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (options) { - int flags_i; + int flags_i = OSTREE_REPO_PULL_FLAGS_NONE; (void) g_variant_lookup (options, "refs", "^a&s", &refs_to_fetch); (void) g_variant_lookup (options, "flags", "i", &flags_i); /* Reduce risk of issues if enum happens to be 64 bit for some reason */ diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 47519e66..d59c8e78 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -3999,11 +3999,17 @@ ostree_repo_pull_one_dir (OstreeRepo *self, * Like ostree_repo_pull(), but supports an extensible set of flags. * The following are currently defined: * - * * subdir (s): Pull just this subdirectory + * * refs (as): Array of string refs * * flags (i): An instance of #OstreeRepoPullFlags - * * refs: (as): Array of string refs - * * depth: (i): How far in the history to traverse; default is 0, -1 means infinite - * * override-commit-ids: (as): Array of specific commit IDs to fetch for refs + * * subdir (s): Pull just this subdirectory + * * override-remote-name (s): If local, add this remote to refspec + * * gpg-verify (b): GPG verify commits + * * gpg-verify-summary (b): GPG verify summary + * * depth (i): How far in the history to traverse; default is 0, -1 means infinite + * * disable-static-deltas (b): Do not use static deltas + * * require-static-deltas (b): Require static deltas + * * override-commit-ids (as): Array of specific commit IDs to fetch for refs + * * dry-run (b): Only print information on what will be downloaded (requires static deltas) */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, From 594162f16a32e944d3bb2fdfea39aced6e6f620b Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 5 Apr 2016 15:37:32 -0400 Subject: [PATCH 56/69] pull: More consistently use remote_repo_local for local repos I think it's cleaner if we use `remote_repo_local` to know that we have a local repo. In reality, it might be nicest if we didn't even create an `OstreeFetcher` for this case, but untangling the code is tricky. Closes: #239 Approved by: alexlarsson --- src/libostree/ostree-repo-pull.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index e86ede50..f292951e 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1990,10 +1990,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, else { pull_data->remote_name = g_strdup (remote_name_or_baseurl); - if (!ostree_repo_remote_get_gpg_verify (self, remote_name_or_baseurl, + if (!ostree_repo_remote_get_gpg_verify (self, pull_data->remote_name, &pull_data->gpg_verify, error)) goto out; - if (!ostree_repo_remote_get_gpg_verify_summary (self, remote_name_or_baseurl, + if (!ostree_repo_remote_get_gpg_verify_summary (self, pull_data->remote_name, &pull_data->gpg_verify_summary, error)) goto out; } @@ -2123,7 +2123,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, } if (bytes_sig && - !_ostree_repo_remote_name_is_file (remote_name_or_baseurl) && + !pull_data->remote_repo_local && !_ostree_repo_load_cache_summary_if_same_sig (self, remote_name_or_baseurl, bytes_sig, @@ -2177,7 +2177,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (!summary_from_cache && bytes_summary && bytes_sig) { - if (!_ostree_repo_remote_name_is_file (remote_name_or_baseurl) && + if (!pull_data->remote_repo_local && !_ostree_repo_cache_summary (self, remote_name_or_baseurl, bytes_summary, From 581a643e17d81200af50f641a4b2c40ceafe46db Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 8 Apr 2016 10:28:17 -0400 Subject: [PATCH 57/69] build: Move grub2-15_ostree back to pkglibexecdir It's not quite namespaced enough to have `/usr/libexec/grub2-15_ostree`, and the Fedora spec file expects things in `/usr/libexec/ostree`. Changing the spec file would be annoying as we'd need conditionals. Closes: #249 Approved by: gatispaeglis --- Makefile-boot.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile-boot.am b/Makefile-boot.am index a2cdc28f..7ec54fa3 100644 --- a/Makefile-boot.am +++ b/Makefile-boot.am @@ -41,10 +41,10 @@ endif if !BUILDOPT_BUILTIN_GRUB2_MKCONFIG # We're using the system grub2-mkconfig generator -libexec_SCRIPTS = src/boot/grub2/grub2-15_ostree +pkglibexec_SCRIPTS += src/boot/grub2/grub2-15_ostree install-grub2-config-hook: mkdir -p $(DESTDIR)$(grub2configdir) - ln -sf $(libexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree + ln -sf $(pkglibexecdir)/grub2-15_ostree $(DESTDIR)$(grub2configdir)/15_ostree grub2configdir = $(sysconfdir)/grub.d INSTALL_DATA_HOOKS += install-grub2-config-hook else From 80c68ba7f0988784b983ce2f1eda875bc325cead Mon Sep 17 00:00:00 2001 From: Gatis Paeglis Date: Tue, 12 Apr 2016 11:26:27 +0200 Subject: [PATCH 58/69] Fix AS_HELP_STRING for builtin grub2 mkconfig Closes: #253 Approved by: giuseppe --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 24dfb356..f995139b 100644 --- a/configure.ac +++ b/configure.ac @@ -252,7 +252,7 @@ AS_IF([test "x$with_dracut" = "xyes" || test "x$with_mkinitcpio" = "xyes"], [ AM_CONDITIONAL(BUILDOPT_SYSTEMD, test x$with_systemd = xyes) AC_ARG_WITH(builtin-grub2-mkconfig, - AS_HELP_STRING([--with-builtin-grub-mkconfig], + AS_HELP_STRING([--with-builtin-grub2-mkconfig], [Use a builtin minimal grub2-mkconfig to generate a GRUB2 configuration file (default: no)]),, [with_builtin_grub2_mkconfig=no]) AM_CONDITIONAL(BUILDOPT_BUILTIN_GRUB2_MKCONFIG, test x$with_builtin_grub2_mkconfig = xyes) From 6a57d0a2f03eddbe44f1d0edaeb5ef32a99d3e69 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 13 Apr 2016 11:14:20 +0200 Subject: [PATCH 59/69] fetcher: Initialize output_stream_set_lock mutex ostree pull-local crashed for me in thread_closure_unref () doing: g_mutex_clear (&thread_closure->output_stream_set_lock); Seems like we never initialize this mutex. Closes: #254 Approved by: cgwalters --- src/libostree/ostree-fetcher.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 26e72c45..d7915ba6 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -541,6 +541,7 @@ _ostree_fetcher_constructed (GObject *object) self->thread_closure->output_stream_set = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) NULL, (GDestroyNotify) g_object_unref); + g_mutex_init (&self->thread_closure->output_stream_set_lock); if (g_getenv ("OSTREE_DEBUG_HTTP")) { From 77ea287cd240825af311f8ac06f272b9a3981b54 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 13 Apr 2016 21:18:19 +0200 Subject: [PATCH 60/69] commit: Fix crash if dfd_iter is NULL in write_directory_content_to_mtree_internal dfd_iter can be NULL, for instance if commiting from --tree=ref=FOO. Don't blindly de-ref it to avoid crashing. Closes: #256 Approved by: cgwalters --- src/libostree/ostree-repo-commit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 0fb3b0fe..7f03e11d 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -2525,7 +2525,7 @@ write_directory_content_to_mtree_internal (OstreeRepo *self, } if (!get_modified_xattrs (self, modifier, - child_relpath, child_info, child, dfd_iter->fd, name, + child_relpath, child_info, child, dfd_iter != NULL ? dfd_iter->fd : -1, name, &xattrs, cancellable, error)) goto out; From b787fce6128d53eee627fea8d49b1363efd15251 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 11 Apr 2016 10:25:40 +0200 Subject: [PATCH 61/69] Add cache_dir_fd to OstreeRepo This will allow us later to easily swap out the cache dir. Closes: #250 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 4 +++- src/libostree/ostree-repo-prune.c | 5 ++++- src/libostree/ostree-repo-pull.c | 26 ++++++++++++++--------- src/libostree/ostree-repo.c | 32 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index c60ab246..1cf99ead 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -33,7 +33,8 @@ G_BEGIN_DECLS #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay" -#define _OSTREE_SUMMARY_CACHE_PATH "tmp/cache/summaries" +#define _OSTREE_SUMMARY_CACHE_DIR "summaries" +#define _OSTREE_CACHE_DIR "cache" /** * OstreeRepo: @@ -52,6 +53,7 @@ struct OstreeRepo { int repo_dir_fd; GFile *tmp_dir; int tmp_dir_fd; + int cache_dir_fd; GFile *objects_dir; GFile *state_dir; int objects_dir_fd; diff --git a/src/libostree/ostree-repo-prune.c b/src/libostree/ostree-repo-prune.c index 9aed0ac3..8c5d13e9 100644 --- a/src/libostree/ostree-repo-prune.c +++ b/src/libostree/ostree-repo-prune.c @@ -125,7 +125,10 @@ _ostree_repo_prune_tmp (OstreeRepo *self, g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; glnx_fd_close int fd = -1; - fd = glnx_opendirat_with_errno (self->repo_dir_fd, _OSTREE_SUMMARY_CACHE_PATH, FALSE); + if (self->cache_dir_fd == -1) + return TRUE; + + fd = glnx_opendirat_with_errno (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, FALSE); if (fd < 0) { if (errno == ENOENT) diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index f292951e..eef5f039 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1783,12 +1783,15 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig"); + const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig"); glnx_fd_close int prev_fd = -1; g_autoptr(GBytes) old_sig_contents = NULL; - if (!ot_openat_ignore_enoent (self->repo_dir_fd, summary_cache_sig_file, &prev_fd, error)) + if (self->cache_dir_fd == -1) + return TRUE; + + if (!ot_openat_ignore_enoent (self->cache_dir_fd, summary_cache_sig_file, &prev_fd, error)) goto out; if (prev_fd < 0) @@ -1803,17 +1806,17 @@ _ostree_repo_load_cache_summary_if_same_sig (OstreeRepo *self, if (g_bytes_compare (old_sig_contents, summary_sig) == 0) { - const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote); + const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote); glnx_fd_close int summary_fd = -1; GBytes *summary_data; - summary_fd = openat (self->repo_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY); + summary_fd = openat (self->cache_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY); if (summary_fd < 0) { if (errno == ENOENT) { - (void) unlinkat (self->repo_dir_fd, summary_cache_sig_file, 0); + (void) unlinkat (self->cache_dir_fd, summary_cache_sig_file, 0); ret = TRUE; goto out; } @@ -1842,13 +1845,16 @@ _ostree_repo_cache_summary (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote); - const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig"); + const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote); + const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_DIR, "/", remote, ".sig"); - if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, _OSTREE_SUMMARY_CACHE_PATH, 0775, cancellable, error)) + if (self->cache_dir_fd == -1) + return TRUE; + + if (!glnx_shutil_mkdir_p_at (self->cache_dir_fd, _OSTREE_SUMMARY_CACHE_DIR, 0775, cancellable, error)) goto out; - if (!glnx_file_replace_contents_at (self->repo_dir_fd, + if (!glnx_file_replace_contents_at (self->cache_dir_fd, summary_cache_file, g_bytes_get_data (summary, NULL), g_bytes_get_size (summary), @@ -1856,7 +1862,7 @@ _ostree_repo_cache_summary (OstreeRepo *self, cancellable, error)) goto out; - if (!glnx_file_replace_contents_at (self->repo_dir_fd, + if (!glnx_file_replace_contents_at (self->cache_dir_fd, summary_cache_sig_file, g_bytes_get_data (summary_sig, NULL), g_bytes_get_size (summary_sig), diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d59c8e78..b8cb2425 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -613,6 +613,8 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->tmp_dir); if (self->tmp_dir_fd) (void) close (self->tmp_dir_fd); + if (self->cache_dir_fd) + (void) close (self->cache_dir_fd); g_clear_object (&self->objects_dir); if (self->objects_dir_fd != -1) (void) close (self->objects_dir_fd); @@ -784,6 +786,7 @@ ostree_repo_init (OstreeRepo *self) g_mutex_init (&self->remotes_lock); self->repo_dir_fd = -1; + self->cache_dir_fd = -1; self->commit_stagedir_fd = -1; self->objects_dir_fd = -1; self->uncompressed_objects_dir_fd = -1; @@ -2371,6 +2374,7 @@ ostree_repo_open (OstreeRepo *self, g_autofree char *version = NULL; g_autofree char *mode = NULL; g_autofree char *parent_repo_path = NULL; + g_autoptr(GError) temp_error = NULL; g_return_val_if_fail (error == NULL || *error == NULL, FALSE); @@ -2507,6 +2511,34 @@ ostree_repo_open (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, "tmp", TRUE, &self->tmp_dir_fd, error)) goto out; + if (!glnx_shutil_mkdir_p_at (self->tmp_dir_fd, _OSTREE_CACHE_DIR, 0775, cancellable, &temp_error)) + { + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_PERMISSION_DENIED)) + { + g_clear_error (&temp_error); + g_debug ("No permissions to create cache dir"); + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + + if (!glnx_opendirat (self->tmp_dir_fd, _OSTREE_CACHE_DIR, TRUE, &self->cache_dir_fd, &temp_error)) + { + if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + { + g_clear_error (&temp_error); + g_debug ("No cache dir"); + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && self->enable_uncompressed_cache) { if (!gs_file_ensure_directory (self->uncompressed_objects_dir, TRUE, cancellable, error)) From 9e7e594907cad21b7f5d843340948c4d426e8680 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 11 Apr 2016 12:43:07 +0200 Subject: [PATCH 62/69] Add OstreeRepo option for an out-of-band cache dir This allows you to have a writable cache dir even for a system-owned repository. Closes: #250 Approved by: cgwalters --- src/libostree/libostree.sym | 1 + src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 30 ++++++++++++++++++++++++++ src/libostree/ostree-repo.h | 7 ++++++ src/ostree/ot-builtin-pull.c | 8 +++++++ src/ostree/ot-remote-builtin-refs.c | 9 ++++++++ src/ostree/ot-remote-builtin-summary.c | 9 ++++++++ tests/test-pull-summary-sigs.sh | 17 ++++++++++++++- 8 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index ea18632f..89a1457e 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -335,4 +335,5 @@ global: ostree_repo_get_remote_option; ostree_repo_get_remote_list_option; ostree_repo_get_remote_boolean_option; + ostree_repo_set_cache_dir; } LIBOSTREE_2016.4; diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index 1cf99ead..a2a99693 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -54,6 +54,7 @@ struct OstreeRepo { GFile *tmp_dir; int tmp_dir_fd; int cache_dir_fd; + char *cache_dir; GFile *objects_dir; GFile *state_dir; int objects_dir_fd; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index b8cb2425..495e2261 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -2573,6 +2573,36 @@ ostree_repo_set_disable_fsync (OstreeRepo *self, self->disable_fsync = disable_fsync; } +/** + * ostree_repo_set_cache_dir: + * @self: An #OstreeRepo + * @dfd: directory fd + * @path: subpath in @dfd + * + * Set a custom location for the cache directory used for e.g. + * per-remote summary caches. Setting this manually is useful when + * doing operations on a system repo as a user because you don't have + * write permissions in the repo, where the cache is normally stored. + */ +gboolean +ostree_repo_set_cache_dir (OstreeRepo *self, + int dfd, + const char *path, + GCancellable *cancellable, + GError **error) +{ + int fd; + + if (!glnx_opendirat (dfd, path, TRUE, &fd, error)) + return FALSE; + + if (self->cache_dir_fd != -1) + close (self->cache_dir_fd); + self->cache_dir_fd = fd; + + return TRUE; +} + /** * ostree_repo_get_disable_fsync: * @self: An #OstreeRepo diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 822014f2..8a04e8e5 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -63,6 +63,13 @@ _OSTREE_PUBLIC void ostree_repo_set_disable_fsync (OstreeRepo *self, gboolean disable_fsync); +_OSTREE_PUBLIC +gboolean ostree_repo_set_cache_dir (OstreeRepo *self, + int dfd, + const char *path, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_get_disable_fsync (OstreeRepo *self); diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 8bef63a3..734f7440 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -35,10 +35,12 @@ static gboolean opt_disable_static_deltas; static gboolean opt_require_static_deltas; static gboolean opt_untrusted; static char* opt_subpath; +static char* opt_cache_dir; static int opt_depth = 0; static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, + { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, @@ -130,6 +132,12 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_disable_fsync) ostree_repo_set_disable_fsync (repo, TRUE); + if (opt_cache_dir) + { + if (!ostree_repo_set_cache_dir (repo, AT_FDCWD, opt_cache_dir, cancellable, error)) + goto out; + } + if (opt_mirror) pullflags |= OSTREE_REPO_PULL_FLAGS_MIRROR; diff --git a/src/ostree/ot-remote-builtin-refs.c b/src/ostree/ot-remote-builtin-refs.c index d21b19cc..da3bcbb9 100644 --- a/src/ostree/ot-remote-builtin-refs.c +++ b/src/ostree/ot-remote-builtin-refs.c @@ -25,7 +25,10 @@ #include "ot-main.h" #include "ot-remote-builtins.h" +static char* opt_cache_dir; + static GOptionEntry option_entries[] = { + { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, }; gboolean @@ -49,6 +52,12 @@ ot_remote_builtin_refs (int argc, char **argv, GCancellable *cancellable, GError goto out; } + if (opt_cache_dir) + { + if (!ostree_repo_set_cache_dir (repo, AT_FDCWD, opt_cache_dir, cancellable, error)) + goto out; + } + remote_name = argv[1]; if (!ostree_repo_remote_list_refs (repo, remote_name, &refs, cancellable, error)) diff --git a/src/ostree/ot-remote-builtin-summary.c b/src/ostree/ot-remote-builtin-summary.c index da76017f..4659dd4d 100644 --- a/src/ostree/ot-remote-builtin-summary.c +++ b/src/ostree/ot-remote-builtin-summary.c @@ -28,7 +28,10 @@ static gboolean opt_raw; +static char* opt_cache_dir; + static GOptionEntry option_entries[] = { + { "cache-dir", 0, 0, G_OPTION_ARG_STRING, &opt_cache_dir, "Use custom cache dir", NULL }, { "raw", 0, 0, G_OPTION_ARG_NONE, &opt_raw, "Show raw variant data", NULL }, { NULL } }; @@ -59,6 +62,12 @@ ot_remote_builtin_summary (int argc, char **argv, GCancellable *cancellable, GEr remote_name = argv[1]; + if (opt_cache_dir) + { + if (!ostree_repo_set_cache_dir (repo, AT_FDCWD, opt_cache_dir, cancellable, error)) + goto out; + } + if (opt_raw) flags |= OSTREE_DUMP_RAW; diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index 40de0a66..65822d23 100755 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -21,7 +21,7 @@ set -euo pipefail . $(dirname $0)/libtest.sh -echo "1..6" +echo "1..7" COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}" setup_fake_remote_repo1 "archive-z2" "${COMMIT_SIGN}" @@ -91,6 +91,21 @@ assert_has_file repo/tmp/cache/summaries/origin assert_has_file repo/tmp/cache/summaries/origin.sig echo "ok prune summary cache" +cd ${test_tmpdir} +repo_reinit +mkdir cachedir +${OSTREE} --repo=repo pull --cache-dir=cachedir origin main +assert_not_has_file repo/tmp/cache/summaries/origin +assert_not_has_file repo/tmp/cache/summaries/origin.sig +assert_has_file cachedir/summaries/origin +assert_has_file cachedir/summaries/origin.sig + +rm cachedir/summaries/origin +${OSTREE} --repo=repo pull --cache-dir=cachedir origin main +assert_not_has_file repo/tmp/cache/summaries/origin +assert_has_file cachedir/summaries/origin + +echo "ok pull with signed summary and cachedir" cd ${test_tmpdir} repo_reinit From d9a334950bcaded268d60511fe23f386bebf0276 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 14 Apr 2016 11:05:22 -0400 Subject: [PATCH 63/69] man: Elaborate on per-remote GPG Closes: #258 Approved by: alexlarsson --- man/ostree.repo-config.xml | 9 +++++++++ man/ostree.xml | 24 ++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index c77ccc6e..0c421ba4 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -195,6 +195,15 @@ Boston, MA 02111-1307, USA. ignored. + + + Per-remote GPG keyrings and verification + + OSTree supports a per-remote GPG keyring. For more information see + ostree1. + in the section GPG verification. + + See Also diff --git a/man/ostree.xml b/man/ostree.xml index 161ef0bc..80b0b0c1 100644 --- a/man/ostree.xml +++ b/man/ostree.xml @@ -425,13 +425,25 @@ Boston, MA 02111-1307, USA. GPG verification - OSTree supports signing commits with GPG. The set of - trusted public keys is stored as keyring files in - /usr/share/ostree/trusted.gpg.d. Any - public key in a keyring file in that directory will be - trusted by the client. No private keys should be present - in this directory. + OSTree supports signing commits with GPG. Operations on the system + repository by default use keyring files in + /usr/share/ostree/trusted.gpg.d. Any + public key in a keyring file in that directory will be + trusted by the client. No private keys should be present + in this directory. + + In addition to the system repository, OSTree supports a + per-remote + remotename.trustedkeys.gpg + file stored in the toplevel of the repository (alongside + objects/ and such). This is + particularly useful when downloading content that may not + be fully trusted (e.g. you want to inspect it but not + deploy it as an OS), or use it for containers. This file + is written via ostree remote add + --gpg-import. + From c86e4f0c9015f1de0a24f66c0cb8c7b93bb1c598 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 8 Apr 2016 16:59:12 +0200 Subject: [PATCH 64/69] Add remotes-config-dir to OstreeRepo This allows you to replace the default $sysroot/$sysconfdir/ostree/repos.d string value, and to use a similar feature for repos that are not the system repo. In particular, this allows us to support /etc/xdg-app/remotes.d for xdg-app. Closes: #247 Approved by: cgwalters --- src/libostree/ostree-repo-private.h | 1 + src/libostree/ostree-repo.c | 43 +++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a2a99693..6a9092e9 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -63,6 +63,7 @@ struct OstreeRepo { int uncompressed_objects_dir_fd; GFile *config_file; GFile *sysroot_dir; + char *remotes_config_dir; GFile *transaction_lock_path; GHashTable *txn_refs; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 495e2261..72a25431 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -88,6 +88,7 @@ enum { PROP_0, PROP_PATH, + PROP_REMOTES_CONFIG_DIR, PROP_SYSROOT_PATH }; @@ -625,6 +626,7 @@ ostree_repo_finalize (GObject *object) (void) close (self->uncompressed_objects_dir_fd); g_clear_object (&self->config_file); g_clear_object (&self->sysroot_dir); + g_free (self->remotes_config_dir); g_clear_object (&self->transaction_lock_path); @@ -664,6 +666,9 @@ ostree_repo_set_property(GObject *object, case PROP_SYSROOT_PATH: self->sysroot_dir = g_value_dup_object (value); break; + case PROP_REMOTES_CONFIG_DIR: + self->remotes_config_dir = g_value_dup_string (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -686,6 +691,9 @@ ostree_repo_get_property(GObject *object, case PROP_SYSROOT_PATH: g_value_set_object (value, self->sysroot_dir); break; + case PROP_REMOTES_CONFIG_DIR: + g_value_set_string (value, self->remotes_config_dir); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -739,6 +747,13 @@ ostree_repo_class_init (OstreeRepoClass *klass) "", G_TYPE_FILE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); + g_object_class_install_property (object_class, + PROP_REMOTES_CONFIG_DIR, + g_param_spec_string ("remotes-config-dir", + "", + "", + NULL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); /** * OstreeRepo::gpg-verify-result: @@ -2315,19 +2330,32 @@ append_one_remote_config (OstreeRepo *self, out: return ret; } - + +static GFile * +get_remotes_d_dir (OstreeRepo *self) +{ + if (self->remotes_config_dir != NULL) + return g_file_resolve_relative_path (self->sysroot_dir, self->remotes_config_dir); + else if (ostree_repo_is_system (self)) + return g_file_resolve_relative_path (self->sysroot_dir, SYSCONF_REMOTES); + + return NULL; +} + static gboolean append_remotes_d (OstreeRepo *self, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - g_autoptr(GFile) etc_ostree_remotes_d = NULL; + g_autoptr(GFile) remotes_d = NULL; g_autoptr(GFileEnumerator) direnum = NULL; - etc_ostree_remotes_d = g_file_resolve_relative_path (self->sysroot_dir, SYSCONF_REMOTES); + remotes_d = get_remotes_d_dir (self); + if (remotes_d == NULL) + return TRUE; - if (!enumerate_directory_allow_noent (etc_ostree_remotes_d, OSTREE_GIO_FAST_QUERYINFO, 0, + if (!enumerate_directory_allow_noent (remotes_d, OSTREE_GIO_FAST_QUERYINFO, 0, &direnum, cancellable, error)) goto out; @@ -2502,11 +2530,8 @@ ostree_repo_open (OstreeRepo *self, ostree_repo_set_disable_fsync (self, TRUE); } - if (ostree_repo_is_system (self)) - { - if (!append_remotes_d (self, cancellable, error)) - goto out; - } + if (!append_remotes_d (self, cancellable, error)) + goto out; if (!glnx_opendirat (self->repo_dir_fd, "tmp", TRUE, &self->tmp_dir_fd, error)) goto out; From 7ac8b0442c022ffc3e200cbcf039455a0fc1dfbf Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 8 Apr 2016 17:00:59 +0200 Subject: [PATCH 65/69] Look for $remotename.trustedkeys.gpg in remotes.d dir This is a nice way to add gpg keys for system configured remotes without making them globally trusted. Closes: #247 Approved by: cgwalters --- src/libostree/ostree-repo.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 72a25431..08e6a48f 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -4575,6 +4575,7 @@ find_keyring (OstreeRepo *self, OstreeRemote *remote, GCancellable *cancellable) { + g_autoptr(GFile) remotes_d = NULL; g_autoptr(GFile) file = NULL; file = g_file_get_child (self->repodir, remote->keyring); @@ -4583,6 +4584,15 @@ find_keyring (OstreeRepo *self, return g_steal_pointer (&file); } + remotes_d = get_remotes_d_dir (self); + if (remotes_d) + { + g_autoptr(GFile) file2 = g_file_get_child (remotes_d, remote->keyring); + + if (g_file_query_exists (file2, cancellable)) + return g_steal_pointer (&file2); + } + if (self->parent_repo) return find_keyring (self->parent_repo, remote, cancellable); From f3796a44586cd6055654d4128ffcec9780673136 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 13 Apr 2016 09:28:30 -0400 Subject: [PATCH 66/69] refs: Add g_prefix_error around opendir for easier debugging Addresses: https://github.com/projectatomic/rpm-ostree/issues/264 We should consider moving this down into `glnx_opendirat`, but for now a quick fix. Closes: #255 Approved by: jlebon --- src/libostree/ostree-repo-refs.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 0c03ad1d..29f03337 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -749,7 +749,10 @@ _ostree_repo_write_ref (OstreeRepo *self, { if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &dfd, error)) - goto out; + { + g_prefix_error (error, "Opening %s: ", "refs/heads"); + goto out; + } } else { @@ -757,7 +760,10 @@ _ostree_repo_write_ref (OstreeRepo *self, if (!glnx_opendirat (self->repo_dir_fd, "refs/remotes", TRUE, &refs_remotes_dfd, error)) - goto out; + { + g_prefix_error (error, "Opening %s: ", "refs/remotes"); + goto out; + } if (rev != NULL) { @@ -767,7 +773,10 @@ _ostree_repo_write_ref (OstreeRepo *self, } if (!glnx_opendirat (refs_remotes_dfd, remote, TRUE, &dfd, error)) - goto out; + { + g_prefix_error (error, "Opening remotes/ dir %s: ", remote); + goto out; + } } if (rev == NULL) From a08b7765b7b909f96ff13b55d2e3e1f0c9c0807c Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 14 Apr 2016 20:55:28 +0200 Subject: [PATCH 67/69] static-delta: Put temp files in /var/tmp We may not have write permissions in the current directory. Closes: #259 Approved by: cgwalters --- src/libostree/ostree-repo-static-delta-compilation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index df5dc19e..b3989234 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -455,7 +455,7 @@ get_unpacked_unlinked_content (OstreeRepo *repo, GError **error) { gboolean ret = FALSE; - g_autofree char *tmpname = g_strdup ("tmpostree-deltaobj-XXXXXX"); + g_autofree char *tmpname = g_strdup ("/var/tmp/tmpostree-deltaobj-XXXXXX"); glnx_fd_close int fd = -1; g_autoptr(GBytes) ret_content = NULL; g_autoptr(GInputStream) istream = NULL; From b4b26907c8e873eae3a35d72f01f349c61585333 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 15 Apr 2016 10:01:42 +0200 Subject: [PATCH 68/69] static-delta: Initialize read_source_fd to -1 If not, we'll get ESPIPE when seeking on fd 0. Closes: #260 Approved by: cgwalters --- src/libostree/ostree-repo-static-delta-processing.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 6c5dd463..681d426f 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -106,6 +106,12 @@ OPPROTO(close) OPPROTO(bspatch) #undef OPPROTO +static void +static_delta_execution_state_init (StaticDeltaExecutionState *state) +{ + state->read_source_fd = -1; +} + static gboolean read_varuint64 (StaticDeltaExecutionState *state, guint64 *out_value, @@ -195,6 +201,8 @@ _ostree_static_delta_part_execute (OstreeRepo *repo, StaticDeltaExecutionState *state = &statedata; guint n_executed = 0; + static_delta_execution_state_init (&statedata); + state->repo = repo; state->async_error = error; state->trusted = trusted; From bfa23bdc1f13a646f1c91f8a2724022eef2d5656 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Apr 2016 10:51:39 -0400 Subject: [PATCH 69/69] Release 2016.5 Closes: #261 Approved by: cgwalters --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f995139b..dca9f536 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([ostree], [2016.4], [walters@verbum.org]) +AC_INIT([ostree], [2016.5], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([buildutil]) AC_CONFIG_AUX_DIR([build-aux])